SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} = \sum_ {i \in \mathcal {S}} \frac {e ^ {- x _ {i} w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}}}{1 + e ^ {- x _ {i} w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}}} \cdot x _ {i} + \sum_ {i \notin \mathcal {S}} \frac {e ^ {- x _ {i} w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}}}{1 + e ^ {- x _ {i} w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}}} \cdot x i \\ \leq \sum_ {i \in \mathcal {S}} \frac {e ^ {- \gamma w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}}}{1 + e ^ {- \gamma w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}}} \cdot \gamma + \sum_ {i \notin \mathcal {S}} \frac {e ^ {- x _ {i} w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}}}{1 + e ^ {- x _ {i} w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}}} \\ \text {s i n c e} x _ {i} = \gamma \text {f o r} i \in \mathcal {S}, \text {a n d} x _ {i} \leq 1 \text {f o r} i \in [ n ] \\ \leq \gamma \cdot \sum_ {i \in \mathcal {S}} \min  \left\{1, e ^ {- \gamma w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}} \right\} + \sum_ {i \notin \mathcal {S}} \min  \left\{1, e ^ {- w _ {t} x _ {i} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}} \right\} \\ \text {s i n c e} e ^ {t} / \left(1 + e ^ {t}\right) \leq \min  \{1, e ^ {t} \} \\ \leq \gamma \cdot \sum_ {i \in \mathcal {S}} \min  \left\{1, e ^ {- \gamma w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}} \right\} + \sum_ {i \notin \mathcal {S}} \min  \left\{1, e ^ {- \theta w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}} \right\} \\ \text {s i n c e} x _ {t} \geq \theta > \gamma \text {f o r} i \notin \mathcal {S} \\ \leq \gamma \cdot \sum_ {i \in S} e ^ {- \gamma w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}} + \sum_ {i \notin S} e ^ {- \theta w _ {t} - \bar {\mathbf {x}} _ {i} ^ {\top} \bar {\mathbf {w}} _ {t}} \\ = \gamma e ^ {- \gamma w _ {t}} \cdot G (\bar {\mathbf {w}} _ {t}) + e ^ {- \theta w _ {t}} \cdot H (\bar {\mathbf {w}} _ {t}). \\ \end{array}
$$
Lemma B.6 (A lower bound on  $w_{t}$ ). Suppose Assumptions 2, 1, and 3 hold. Then it holds that
$$
w _ {t} \geq \frac {1}{\gamma} \cdot \log \left(1 + \frac {\eta \gamma^ {2}}{2} \cdot t\right), \quad t \geq 0.
$$
As a direct consequence, it holds that
$$
e ^ {- \gamma w _ {t}} \leq \frac {2}{2 + \eta \gamma^ {2} \cdot t}, \quad t \geq 0.
$$
Proof. Observe that
$$
\begin{array}{l} w _ {t + 1} \geq w _ {t} + \frac {\eta \gamma}{2} \cdot \min  \left\{1, e ^ {- \gamma w _ {t}} \cdot G (\bar {\mathbf {w}} _ {t}) \right\} \quad \text {b y} \\ \geq w _ {t} + \frac {\eta \gamma}{2} \cdot \min  \left\{1, e ^ {- \gamma w _ {t}} \cdot 1 \right\} \quad \text {b y} \\ \geq w _ {t} + \frac {\eta \gamma}{2} \cdot e ^ {- \gamma w _ {t}}, \quad \text {s i n c e} w _ {t} \geq 0 \text {b y L e m m a B . 1} \tag {20} \\ \end{array}
$$
which implies that  $w_{t}$  is increasing. Furthermore, we have
$$
\begin{array}{l} \begin{array}{l} e ^ {\gamma w _ {t + 1}} - e ^ {\gamma w _ {t}} = e ^ {\gamma w _ {t}} \cdot \left(e ^ {\gamma (w _ {t + 1} - w _ {t})} - 1\right) \\ \qquad \geq e ^ {\gamma w _ {t}} \cdot \gamma (w _ {t + 1} - w _ {t}) \qquad \text {s i n c e} e ^ {t} - 1 \geq t \text {f o r} t \geq 0, \text {a n d} w _ {t + 1} \geq w _ {t} \end{array} \\ \geq \frac {\eta \gamma^ {2}}{2}, \quad \text {b y} (2 0) \\ \end{array}
$$
$$
\begin{array}{r l} e ^ {\gamma w _ {t}} \geq e ^ {\gamma w _ {0}} + \frac {\eta \gamma^ {2}}{2} \cdot t \\ = 1 + \frac {\eta \gamma^ {2}}{2} \cdot t. \end{array} \qquad \text {s i n c e} w _ {0} = 0
$$
Lemma B.7 (An upper bound on  $w_{t}$ ). Suppose Assumptions 1, 2, and 3 hold. Then it holds that
$$
w _ {t} \leq \frac {1}{\gamma} \cdot \log \left(\left(e \eta \gamma^ {2} G _ {\max } + e \eta \gamma H _ {\max }\right) \cdot (t + 1)\right), \quad t \geq 0.
$$
$$
e ^ {- \gamma w _ {t}} \geq \frac {1}{\left(e \eta \gamma^ {2} G _ {\max } + e \eta \gamma H _ {\max }\right) \cdot (t + 1)}, \quad t \geq 0.
$$
$$
\begin{array}{l} w _ {t + 1} - w _ {t} \leq \eta \gamma \cdot e ^ {- \gamma w _ {t}} \cdot G (\bar {\mathbf {w}} _ {t}) + \eta \cdot e ^ {- \theta w _ {t}} \cdot H (\bar {\mathbf {w}} _ {t}) \quad \text {b y} \\ \leq \eta \gamma \cdot e ^ {- \gamma w _ {t}} \cdot G (\bar {\mathbf {w}} _ {t}) + \eta \cdot e ^ {- \gamma w _ {t}} \cdot H (\bar {\mathbf {w}} _ {t}) \quad \text {s i n c e} \theta > \gamma \text {b y} \\ \leq \eta \cdot \left(\gamma G _ {\max } + H _ {\max }\right) \cdot e ^ {- \gamma w _ {t}} \quad \text {b y} \tag {21} \\ \end{array}
$$
$$
t _ {0} := \min  \left\{t: \gamma \eta \cdot \left(\gamma G _ {\max } + H _ {\max }\right) \cdot e ^ {- \gamma w _ {t}} \leq 1 \right\}.
$$
Recall that  $w_{t}$  is increasing according to (20). So we have
$$
\text {f o r} t \leq t _ {0}, \quad w _ {t} \leq \frac {1}{\gamma} \cdot \log (\eta \gamma^ {2} G _ {\max } + \eta \gamma H _ {\max }); \tag {22}
$$
$$
\text {f o r} t \geq t _ {0}, \quad \gamma \eta \cdot \left(\gamma G _ {\max } + H _ {\max }\right) \cdot e ^ {- \gamma w _ {t}} \leq 1. \tag {23}
$$
(21) and (23) together imply that
$$
\text {f o r} t \geq t _ {0}, \quad 0 \leq \gamma \cdot \left(w _ {t + 1} - w _ {t}\right) \leq 1. \tag {24}
$$
Then for  $t \geq t_0$ , we have
$$
\begin{array}{l} e ^ {\gamma w _ {t + 1}} - e ^ {\gamma w _ {t}} = e ^ {\gamma w _ {t}} \left(e ^ {\gamma \left(w _ {t + 1} - w _ {t}\right)} - 1\right) \\ \leq e ^ {\gamma w _ {t}} \cdot e \cdot \gamma (w _ {t + 1} - w _ {t}) \quad \text {b y (2 4) a n d t h a t} e ^ {t} - 1 \leq e \cdot t \text {f o r} 0 \leq t \leq 1 \\ \leq e \eta \gamma^ {2} G _ {\max } + e \eta \gamma H _ {\max }, \quad \text {b y} (2 1). \\ \end{array}
$$
$$
\begin{array}{l} e ^ {\gamma w _ {t}} \leq e ^ {\gamma w _ {t _ {0}}} + \left(e \eta \gamma^ {2} G _ {\max } + e \eta \gamma H _ {\max }\right) \cdot (t - t _ {0}) \\ \leq \eta \gamma^ {2} G _ {\max } + \eta \gamma H _ {\max } + \left(e \eta \gamma^ {2} G _ {\max } + e \eta \gamma H _ {\max }\right) \cdot (t - t _ {0}) \quad \text {b y} \tag {22} \\ \leq \left(e \eta \gamma^ {2} G _ {\max } + e \eta \gamma H _ {\max }\right) \cdot (t + 1). \\ \end{array}
$$
Therefore, for  $t \geq t_0$ , we have
$$
w _ {t} \leq \frac {1}{\gamma} \cdot \log \left(\left(e \eta \gamma^ {2} G _ {\max } + e \eta \gamma H _ {\max }\right) \cdot (t + 1)\right).
$$
Note that the above also holds for  $0 \leq t \leq t_0$  according to (22). We have completed the proof.
B.4 Convergence of GD in the Non-Separable Subspace
We show that the vanilla gradient on the non-separable subspace,  $\nabla_{\bar{\mathbf{w}}}L(w_t,\bar{\mathbf{w}}_t)$ , can be understood as the gradient on a modified loss with a rescaling factor,  $e^{-\gamma w_t}\nabla G(\bar{\mathbf{w}}_t)$ , ignoring higher order errors.
Lemma B.8 (Gradients comparison lemma). Suppose Assumptions 1, 2, and 3 hold. Then it holds that
$$
\left\| \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) - e ^ {- \gamma w _ {t}} \cdot \nabla G (\bar {\mathbf {w}} _ {t}) \right\| _ {2} \leq e ^ {- 2 \gamma w _ {t}} \cdot G _ {\max } ^ {2} + e ^ {- \theta w _ {t}} \cdot H _ {\max }, \quad t \geq 0.
$$
As a direct consequence, for every vector  $\bar{\mathbf{v}}\in \mathbb{R}^{d - 1}$ , it holds that
$$
\left\langle \bar {\mathbf {v}}, \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) \right\rangle \leq e ^ {- \gamma w _ {t}} \cdot \left\langle \bar {\mathbf {v}}, \nabla G (\bar {\mathbf {w}} _ {t}) \right\rangle + \| \bar {\mathbf {v}} \| _ {2} \cdot \left(e ^ {- 2 \gamma w _ {t}} \cdot G _ {\max } ^ {2} + e ^ {- \theta w _ {t}} \cdot H _ {\max }\right).
$$
$$
\nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) = - \sum_ {i = 1} ^ {n} \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot \bar {\mathbf {x}} _ {i} \qquad \nabla G (\bar {\mathbf {w}} _ {t}) = - \sum_ {i \in \mathcal {S}} e ^ {- \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \cdot \bar {\mathbf {x}} _ {i}.
$$
By the triangle inequality, we have
$$
\begin{array}{l} \left\| \nabla_ {\bar {\mathbf {w}}} L \left(w _ {t}, \bar {\mathbf {w}} _ {t}\right) - e ^ {- \gamma w _ {t}} \nabla G (\bar {\mathbf {w}} _ {t}) \right\| _ {2} \\ = \left\| \sum_ {i \in S} \left(e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} - \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}\right) \cdot \bar {\mathbf {x}} _ {i} - \sum_ {i \notin S} \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot \bar {\mathbf {x}} _ {i} \right\| _ {2} \\ \leq \underbrace {\left\| \sum_ {i \in \mathcal {S}} \left(e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} - \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}\right) \cdot \bar {\mathbf {x}} _ {i} \right\| _ {2}} _ {(\clubsuit)} + \underbrace {\left\| \sum_ {i \notin \mathcal {S}} \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot \bar {\mathbf {x}} _ {i} \right\| _ {2}} _ {(\odot)}. \tag {25} \\ \end{array}
$$
The  $(\clubsuit)$  term can be bounded by
$$
\begin{array}{l} (\clubsuit) = \left\| \sum_ {i \in \mathcal {S}} \left(e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} - \frac {e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}\right) \cdot \bar {\mathbf {x}} _ {i} \right\| _ {2} \quad \text {s i n c e} x _ {i} = \gamma \text {f o r} i \in \mathcal {S} \\ = \left\| \sum_ {i \in \mathcal {S}} \frac {e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \cdot \bar {\mathbf {x}} _ {i} \right\| _ {2} \\ = e ^ {- 2 \gamma w _ {t}} \cdot \left\| \sum_ {i \in S} \frac {1}{1 + e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot e ^ {- 2 \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \cdot \bar {\mathbf {x}} _ {i} \right\| _ {2} \\ \leq e ^ {- 2 \gamma w _ {t}} \cdot \sum_ {i \in \mathcal {S}} \frac {1}{1 + e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot e ^ {- 2 \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \cdot \| \bar {\mathbf {x}} _ {i} \| _ {2} \quad \text {b y} \\ \leq e ^ {- 2 \gamma w _ {t}} \cdot \sum_ {i \in S} \frac {1}{1 + e ^ {- \gamma w _ {t} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot e ^ {- 2 \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \quad \text {s i n c e} \| \bar {\mathbf {x}} _ {i} \| _ {2} \leq 1 \text {b y A s s u m p t i o n 2} \\ \leq e ^ {- 2 \gamma w _ {t}} \cdot \sum_ {i \in S} e ^ {- 2 \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \\ \leq e ^ {- 2 \gamma w _ {t}} \cdot \left(\sum_ {i \in \mathcal {S}} e ^ {- \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}\right) ^ {2} \\ = e ^ {- 2 \gamma w _ {t}} \cdot G (\bar {\mathbf {w}} _ {t}) ^ {2} \\ \leq e ^ {- 2 \gamma w _ {t}} \cdot G _ {\max } ^ {2}. \quad \text {b y} \\ \end{array}
$$
The  $(\heartsuit)$  term can be bounded by
$$
\begin{array}{l} \left(\heartsuit\right) = \left\| \sum_ {i \notin \mathcal {S}} \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot \bar {\mathbf {x}} _ {i} \right\| _ {2} \\ \leq \sum_ {i \notin \mathcal {S}} \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot \| \bar {\mathbf {x}} _ {i} \| _ {2} \quad \text {b y} \\ \leq \sum_ {i \notin S} \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \quad \text {s i n c e} \| \bar {\mathbf {x}} _ {i} \| _ {2} \leq 1 \text {b y A s s u m p t i o n 2} \\ \leq \sum_ {i \notin S} e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \\ \leq e ^ {- \theta w _ {t}} \cdot \sum_ {i \notin \mathcal {S}} e ^ {- \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \quad x _ {i} \geq \theta > \gamma \text {f o r} i \notin \mathcal {S} \\ = e ^ {- \theta w _ {t}} \cdot H (\bar {\mathbf {w}}) \\ \end{array}
$$
$$
\leq e ^ {- \theta w _ {t}} \cdot H _ {\max}.
$$
by Definition 3
Bringing the bounds on the  $(\clubsuit)$  and  $(\heartsuit)$  into (25), we obtain
We have shown the first conclusion. The second conclusion follows from the first conclusion: for every  $\mathbf{v} \in \mathbb{R}^{d-1}$ ,
$$
\begin{array}{l} \langle \bar {\mathbf {v}}, \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) \rangle = e ^ {- \gamma w _ {t}} \cdot \langle \bar {\mathbf {v}}, \nabla G (\bar {\mathbf {w}} _ {t}) \rangle + \langle \bar {\mathbf {v}}, \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) - e ^ {- \gamma w _ {t}} \cdot \nabla G (\bar {\mathbf {w}} _ {t}) \rangle \\ \leq e ^ {- \gamma w _ {t}} \cdot \langle \bar {\mathbf {v}}, \nabla G (\bar {\mathbf {w}} _ {t}) \rangle + \| \bar {\mathbf {v}} \| _ {2} \cdot \| \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) - e ^ {- \gamma w _ {t}} \cdot \nabla G (\bar {\mathbf {w}} _ {t}) \| _ {2} \\ \leq e ^ {- \gamma w _ {t}} \cdot \left\langle \bar {\mathbf {v}}, \nabla G (\bar {\mathbf {w}} _ {t}) \right\rangle + \| \bar {\mathbf {v}} \| _ {2} \cdot \left(e ^ {- 2 \gamma w _ {t}} \cdot G _ {\max } ^ {2} + e ^ {- \theta w _ {t}} \cdot H _ {\max }\right). \\ \end{array}
$$
Lemma B.9 (A gradient norm bound). Suppose Assumptions 1, 2, and 3 hold. Then it holds that
$$
\left\| \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) \right\| _ {2} \leq e ^ {- \gamma w _ {t}} \cdot (G _ {\max} + H _ {\max}), t \geq 0.
$$
Proof. The inequality is because:
$$
\begin{array}{l} \left\| \nabla_ {\bar {\mathbf {w}} _ {t}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) \right\| _ {2} = \left\| \sum_ {i = 1} ^ {n} \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot \bar {\mathbf {x}} _ {i} \right\| _ {2} \\ \leq \sum_ {i = 1} ^ {n} \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \cdot \| \bar {\mathbf {x}} _ {i} \| _ {2} \quad \text {b y} \\ \leq \sum_ {i = 1} ^ {n} \frac {e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}}{1 + e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}}} \quad \text {s i n c e} \| \bar {\mathbf {x}} _ {i} \| _ {2} \leq 1 \text {b y A s s u m p t i o n 2} \\ \leq \sum_ {i = 1} ^ {n} e ^ {- w _ {t} x _ {i} - \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \\ \leq e ^ {- \gamma w _ {t}} \cdot \sum_ {i = 1} ^ {n} e ^ {- \bar {\mathbf {w}} _ {t} ^ {\top} \bar {\mathbf {x}} _ {i}} \quad \text {s i n c e} x _ {i} \geq \gamma \text {f o r} i \in [ n ] \\ = e ^ {- \gamma w _ {t}} \cdot \left(G (\bar {\mathbf {w}} _ {t}) + H (\bar {\mathbf {w}} _ {t})\right) \\ \leq e ^ {- \gamma w _ {t}} \cdot \left(G _ {\max } + H _ {\max }\right). \\ \end{array}
$$
The next lemma shows that the function value is "non-increasing" ignoring higher order terms.
Lemma B.10 (A modified descent lemma). Suppose Assumptions 1, 2, and 3 hold. Then it holds that
$$
G (\bar {\mathbf {w}} _ {t + 1}) \leq G (\bar {\mathbf {w}} _ {t}) + 2 (\eta + \eta^ {2}) \cdot G _ {\max } \cdot \left(G _ {\max } ^ {2} + H _ {\max } ^ {2}\right) \cdot \left(e ^ {- 2 \gamma w _ {t}} + e ^ {- \theta w _ {t}}\right), \quad t \geq 0.
$$
As a direct consequence of the above and Lemma B.6, it holds that
$$
G (\bar {\mathbf {w}} _ {t + k}) \leq G (\bar {\mathbf {w}} _ {t}) + c _ {0} \cdot 2 (1 + \eta) G _ {\max } \cdot \left((t - 1) ^ {- 1} + (t - 1) ^ {1 - \frac {\theta}{\gamma}}\right), \quad k \geq 0, t \geq 1,
$$
where  $\theta / \gamma > 1$  by Definition 1 and  $c_0$  is a polynomial on  $\{e^\eta, e^n, e^{1/b}, \frac{1}{\eta}, \frac{1}{\theta - \gamma}, \frac{1}{\gamma}, e^{\theta/\gamma}\}$  and is independent of  $t$ , given by
$$
c _ {0} := \eta \cdot \left(G _ {\max } ^ {2} + H _ {\max } ^ {2}\right) \cdot \frac {\theta}{\theta - \gamma} \cdot \max  \left\{\left(\frac {2}{\eta \gamma^ {2}}\right) ^ {2}, \left(\frac {2}{\eta \gamma^ {2}}\right) ^ {\theta / \gamma} \right\}.
$$
$$
\| \nabla^ {2} G (\bar {\mathbf {w}}) \| _ {2} = \left\| \sum_ {i \in \mathcal {S}} e ^ {- \bar {\mathbf {w}} ^ {\top} \bar {\mathbf {x}} _ {i}} \mathbf {x} _ {i} \mathbf {x} _ {i} ^ {\top} \right\| _ {2}
$$
$$
\begin{array}{l} \leq \sum_ {i \in \mathcal {S}} e ^ {- \bar {\mathbf {w}} ^ {\top} \bar {\mathbf {x}} _ {i}} \| \mathbf {x} _ {i} \| _ {2} ^ {2} \quad \text {b y} \\ \leq \sum_ {i \in S} e ^ {- \bar {\mathbf {w}} ^ {\top} \bar {\mathbf {x}} _ {i}} \quad \text {s i n c e} \| \bar {\mathbf {x}} _ {i} \| _ {2} \leq 1 \text {b y A s s u m p t i o n 2} \\ = G (\bar {\mathbf {w}}). \\ \end{array}
$$
Recall that  $\| \bar{\mathbf{w}}_t\| _2\leq W_{\mathrm{max}}$  . So we have
$$
\sup  _ {t} \| \nabla^ {2} G (\bar {\mathbf {w}} _ {t}) \| _ {2} \leq \sup  _ {\| \bar {\mathbf {w}} \| _ {2} \leq W _ {\max }} \| \nabla^ {2} G (\bar {\mathbf {w}}) \| _ {2} \leq \sup  _ {\| \bar {\mathbf {w}} \| _ {2} \leq W _ {\max }} G (\bar {\mathbf {w}}) =: G _ {\max } \tag {26}
$$
Then we can apply Taylor's theorem to obtain that
$$
\begin{array}{l} G \left(\bar {\mathbf {w}} _ {t + 1}\right) \leq G \left(\bar {\mathbf {w}} _ {t}\right) + \left\langle \nabla G \left(\bar {\mathbf {w}} _ {t}\right), \bar {\mathbf {w}} _ {t + 1} - \bar {\mathbf {w}} _ {t} \right\rangle + \frac {G _ {\max }}{2} \cdot \left\| \bar {\mathbf {w}} _ {t + 1} - \bar {\mathbf {w}} _ {t} \right\| _ {2} ^ {2} \tag {26} \\ = G (\bar {\mathbf {w}} _ {t}) - \eta \cdot \langle \nabla G (\bar {\mathbf {w}} _ {t}), \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) \rangle + \frac {G _ {\max}}{2} \cdot \| \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) \| _ {2} ^ {2}. \\ \end{array}
$$
Next we use Lemma B.8 with  $\mathbf{v} = -\nabla G(\bar{\mathbf{w}}_t)$  to get The cross-term is bounded by
$$
\begin{array}{l} - \left\langle \nabla G (\bar {\mathbf {w}} _ {t}), \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) \right\rangle \\ \leq - e ^ {- \gamma w _ {t}} \cdot \left\| \nabla G (\bar {\mathbf {w}} _ {t}) \right\| _ {2} ^ {2} + \left\| \nabla G (\bar {\mathbf {w}} _ {t}) \right\| _ {2} \cdot \left(e ^ {- 2 \gamma w _ {t}} \cdot G _ {\max } ^ {2} + e ^ {- \theta w _ {t}} \cdot H _ {\max }\right) \\ \leq - e ^ {- \gamma w _ {t}} \cdot \left\| \nabla G (\bar {\mathbf {w}} _ {t}) \right\| _ {2} ^ {2} + G _ {\max } \cdot \left(e ^ {- 2 \gamma w _ {t}} \cdot G _ {\max } ^ {2} + e ^ {- \theta w _ {t}} \cdot H _ {\max }\right). \tag {26} \\ \end{array}
$$
Using the above and the gradient norm bound from Lemma B.9, we get that
$$
\begin{array}{l} G (\bar {\mathbf {w}} _ {t + 1}) \leq G (\bar {\mathbf {w}} _ {t}) - \eta e ^ {- \gamma w _ {t}} \cdot \left\| \nabla G (\bar {\mathbf {w}} _ {t}) \right\| _ {2} ^ {2} \\ + \eta e ^ {- 2 \gamma w _ {t}} \cdot G _ {\max } ^ {3} + \eta e ^ {- \theta w _ {t}} \cdot G _ {\max } \cdot H _ {\max } + \eta^ {2} e ^ {- 2 \gamma w _ {t}} \cdot \left(G _ {\max } + H _ {\max }\right) ^ {2} \\ \leq G (\bar {\mathbf {w}} _ {t}) + \eta e ^ {- 2 \gamma w _ {t}} \cdot G _ {\max } ^ {3} + \eta e ^ {- \theta w _ {t}} \cdot G _ {\max } \cdot H _ {\max } + \eta^ {2} e ^ {- 2 \gamma w _ {t}} \cdot (G _ {\max } + H _ {\max }) ^ {2} \\ \leq G (\bar {\mathbf {w}} _ {t}) + 2 (\eta + \eta^ {2}) \cdot G _ {\max } \cdot \left(G _ {\max } ^ {2} + H _ {\max } ^ {2}\right) \cdot \left(e ^ {- 2 \gamma w _ {t}} + e ^ {- \theta w _ {t}}\right), \\ \end{array}
$$
where in the last inequality we use that  $G_{\max} \geq G_{\min} \geq 1$  by Definition 3.
From the above we have
$$
G \left(\bar {\mathbf {w}} _ {t + k}\right) \leq G \left(\bar {\mathbf {w}} _ {t}\right) + 2 \left(\eta + \eta^ {2}\right) \cdot G _ {\max } \cdot \left(G _ {\max } ^ {2} + H _ {\max } ^ {2}\right) \cdot \sum_ {s = t} ^ {s + k} \left(e ^ {- 2 \gamma w _ {s}} + e ^ {- \theta w _ {s}}\right). \tag {27}
$$
The summation is small by Lemma B.6, because
$$
\begin{array}{l} \sum_ {s = t} ^ {s + k} \left(e ^ {- 2 \gamma w _ {s}} + e ^ {- \theta w _ {s}}\right) \\ \leq \sum_ {s = t} ^ {s + k} \left(\frac {2}{2 + \eta \gamma^ {2} \cdot s}\right) ^ {2} + \sum_ {s = t} ^ {s + k} \left(\frac {2}{2 + \eta \gamma^ {2} \cdot s}\right) ^ {\frac {\theta}{\gamma}} \quad \text {b y} \\ \leq \left(\frac {2}{\eta \gamma^ {2}}\right) ^ {2} \cdot \sum_ {s = t} ^ {s + k} s ^ {- 2} + \left(\frac {2}{\eta \gamma^ {2}}\right) ^ {\frac {\theta}{\gamma}} \cdot \sum_ {s = t} ^ {s + k} s ^ {- \frac {\theta}{\gamma}} \\ \leq \left(\frac {2}{\eta \gamma^ {2}}\right) ^ {2} \cdot (t - 1) ^ {- 1} + \left(\frac {2}{\eta \gamma^ {2}}\right) ^ {\frac {\theta}{\gamma}} \cdot \frac {(t - 1) ^ {1 - \frac {\theta}{\gamma}}}{\frac {\theta}{\gamma} - 1} \quad \text {b y} \\ \leq \max  \left\{\left(\frac {2}{\eta \gamma^ {2}}\right) ^ {2}, \left(\frac {2}{\eta \gamma^ {2}}\right) ^ {\theta / \gamma} \right\} \cdot \frac {\theta}{\theta - \gamma} \cdot \left((t - 1) ^ {- 1} + (t - 1) ^ {1 - \frac {\theta}{\gamma}}\right). \\ \end{array}
$$
Inserting the above into (27) completes the proof.
We now prove the convergence of the iterates projected on the non-separable subspace.
Lemma B.11 (Convergence on the non-separable subspace). Suppose Assumptions 1, 2, and 3 hold. Then it holds that
$$
G (\bar {\mathbf {w}} _ {T}) - G (\bar {\mathbf {w}} _ {*}) \leq \frac {c _ {1}}{\log (T)}, \quad T \geq 3,
$$
where  $c_{1} > 0$  is a polynomial on  $\{e^{\eta}, e^{n}, e^{1/b}, \frac{1}{\eta}, \frac{1}{\theta - \gamma}, \frac{1}{\gamma}, e^{\theta/\gamma}\}$  and is independent of  $T$ .
Proof. The proof is conducted in several steps.
Step 1: one-step function value bound. Observe that
$$
\begin{array}{l} \left\| \bar {\mathbf {w}} _ {t + 1} - \bar {\mathbf {w}} _ {*} \right\| _ {2} ^ {2} = \left\| \bar {\mathbf {w}} _ {t} - \bar {\mathbf {w}} _ {*} \right\| _ {2} ^ {2} + 2 \cdot \left\langle \bar {\mathbf {w}} _ {t} - \bar {\mathbf {w}} _ {*}, \bar {\mathbf {w}} _ {t + 1} - \bar {\mathbf {w}} _ {t} \right\rangle + \left\| \bar {\mathbf {w}} _ {t + 1} - \bar {\mathbf {w}} _ {t} \right\| _ {2} ^ {2} \\ = \| \bar {\mathbf {w}} _ {t} - \bar {\mathbf {w}} _ {*} \| _ {2} ^ {2} - 2 \eta \cdot \left\langle \bar {\mathbf {w}} _ {t} - \bar {\mathbf {w}} _ {*}, \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) \right\rangle + \eta^ {2} \cdot \| \nabla_ {\bar {\mathbf {w}}} L (w _ {t}, \bar {\mathbf {w}} _ {t}) \| _ {2} ^ {2}. \\ \end{array}
$$
For the cross-term, we apply Lemma B.8 with  $\mathbf{v} = -\left(\bar{\mathbf{w}}_t - \bar{\mathbf{w}}_*\right)$  to obtain