SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \left\langle \mathbf {w} _ {j, r} ^ {(t, b)} - \mathbf {w} _ {j, r} ^ {(0, 0)}, \boldsymbol {\mu} \right\rangle = j \cdot \gamma_ {j, r} ^ {(t, b)} + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \bar {\rho} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \left\langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\mu} \right\rangle \\ + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\mu} \rangle . \\ \end{array}
$$
According to Lemma B.1, we have
$$
\begin{array}{l} \left| \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \bar {\rho} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\mu} \rangle + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\mu} \rangle \right| \\ \leq \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} | \overline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} | \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot | \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\mu} \rangle | + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} | \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} | \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot | \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\mu} \rangle | \\ \leq \frac {2 \| \boldsymbol {\mu} \| _ {2} \sqrt {2 \log (6 n / \delta)}}{(P - 1) \sigma_ {p} d} \left(\sum_ {i ^ {\prime} = 1} ^ {n} | \bar {\rho} _ {j, r, i ^ {\prime}} ^ {(t, b)} | + \sum_ {i ^ {\prime} = 1} ^ {n} | \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} |\right) \\ \end{array}
$$
$$
\begin{array}{l} = \operatorname {S N R} \sqrt {\frac {8 \log (6 n / \delta)}{d}} \left(\sum_ {i ^ {\prime} = 1} ^ {n} | \bar {\rho} _ {j, r, i ^ {\prime}} ^ {(t, b)} | + \sum_ {i ^ {\prime} = 1} ^ {n} | \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} |\right) \\ \leq \mathrm {S N R} \sqrt {\frac {3 2 \log (6 n / \delta)}{d}} n \alpha , \\ \end{array}
$$
where the first inequality is by triangle inequality, the second inequality is by Lemma B.1, the equality is by  $\mathrm{SNR} = \| \pmb {\mu}\| _2 / ((P - 1)\sigma_p\sqrt{d})$  , and the last inequality is by (27), (28). It follows that
$$
\big | \langle \mathbf {w} _ {j, r} ^ {(t, b)} - \mathbf {w} _ {j, r} ^ {(0, 0)}, \boldsymbol {\mu} \rangle - j \cdot \gamma_ {j, r} ^ {(t, b)} \big | \leq \mathrm {S N R} \sqrt {\frac {3 2 \log (6 n / \delta)}{d}} n \alpha .
$$
Then, for  $j \neq y_{i}$  and any  $t \geq 0$ , we have
$$
\begin{array}{l} \left\langle \mathbf {w} _ {j, r} ^ {(t, b)} - \mathbf {w} _ {j, r} ^ {(0, 0)}, \pmb {\xi} _ {i} \right\rangle \\ = j \cdot \gamma_ {j, r} ^ {(t, b)} \| \boldsymbol {\mu} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \overline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle \\ + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \pmb {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \pmb {\xi} _ {i ^ {\prime}}, \pmb {\xi} _ {i} \rangle \\ = j \cdot \gamma_ {j, r} ^ {(t, b)} \| \boldsymbol {\mu} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle \\ = \frac {1}{P - 1} \underline {{\rho}} _ {j, r, i} ^ {(t, b)} + j \cdot \gamma_ {j, r} ^ {(t, b)} \| \boldsymbol {\mu} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle + \frac {1}{P - 1} \sum_ {i ^ {\prime} \neq i} \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle , \\ \end{array}
$$
where the second equality is due to  $\underline{\rho}_{j,r,i}^{(t,b)} = 0$  for  $j\neq y_{i}$ . Next, we have
$$
\begin{array}{l} \left| j \cdot \gamma_ {j, r} ^ {(t, b)} \| \boldsymbol {\mu} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle + \frac {1}{P - 1} \sum_ {i ^ {\prime} \neq i} \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle \right| \\ \leq | \gamma_ {j, r} ^ {(t, b)} | \| \boldsymbol {\mu} \| _ {2} ^ {- 2} \cdot | \langle \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle | + \frac {1}{P - 1} \sum_ {i ^ {\prime} \neq i} | \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} | \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot | \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle | \\ \leq \left| \gamma_ {j, r} ^ {(t, b)} \right| \left\| \boldsymbol {\mu} \right\| _ {2} ^ {- 1} \sigma_ {p} \sqrt {2 \log (6 n / \delta)} + \frac {4}{P - 1} \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} \sum_ {i ^ {\prime} \neq i} \left| \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \right| \\ = \frac {\mathrm {S N R} ^ {- 1}}{P - 1} \sqrt {\frac {2 \log (6 n / \delta)}{d}} | \gamma_ {j, r} ^ {(t, b)} | + \frac {4}{P - 1} \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} \sum_ {i ^ {\prime} \neq i} | \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} | \\ \leq \frac {\mathrm {S N R}}{P - 1} \sqrt {\frac {8 C ^ {2} \log (6 n / \delta)}{d}} n \alpha + \frac {4}{P - 1} \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} n \alpha \\ \leq \frac {5}{P - 1} \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} n \alpha , \\ \end{array}
$$
where the first inequality is by triangle inequality; the second inequality is by Lemma B.1; the equality is by  $\mathrm{SNR} = \|\pmb{\mu}\|_2 / \sigma_p\sqrt{d}$ ; the third inequality is by (28) and (29); the forth inequality is by  $\mathrm{SNR} \leq 1 / \sqrt{8C'^2}$ . Therefore, for  $j \neq y_i$ , we have
$$
\left| \left\langle \mathbf {w} _ {j, r} ^ {(t, b)} - \mathbf {w} _ {j, r} ^ {(0, 0)}, \boldsymbol {\xi} _ {i} \right\rangle - \frac {1}{P - 1} \rho_ {j, r, i} ^ {(t, b)} \right| \leq \frac {5}{P - 1} \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} n \alpha .
$$
Similarly, we have for  $y_{i} = j$  that
$$
\begin{array}{l} \left\langle \mathbf {w} _ {j, r} ^ {(t, b)} - \mathbf {w} _ {j, r} ^ {(0, 0)}, \boldsymbol {\xi} _ {i} \right\rangle \\ = j \cdot \gamma_ {j, r} ^ {(t, b)} \| \boldsymbol {\mu} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \bar {\rho} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle \\ \end{array}
$$
$$
\begin{array}{l} + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \underline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle \\ = j \cdot \gamma_ {j, r} ^ {(t, b)} \| \boldsymbol {\mu} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle + \frac {1}{P - 1} \sum_ {i ^ {\prime} = 1} ^ {n} \bar {\rho} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle \\ = \frac {1}{P - 1} \bar {\rho} _ {j, r, i} ^ {(t, b)} + j \cdot \gamma_ {j, r} ^ {(t, b)} \| \boldsymbol {\mu} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle + \frac {1}{P - 1} \sum_ {i ^ {\prime} \neq i} \bar {\rho} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle , \\ \end{array}
$$
$$
\begin{array}{l} \left| j \cdot \gamma_ {j, r} ^ {(t, b)} \| \boldsymbol {\mu} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle + \sum_ {i ^ {\prime} \neq i} \overline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} \| \boldsymbol {\xi} _ {i ^ {\prime}} \| _ {2} ^ {- 2} \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle \right| \\ \leq \frac {\mathrm {S N R} ^ {- 1}}{P - 1} \sqrt {\frac {2 \log (6 n / \delta)}{d}} | \gamma_ {j, r} ^ {(t, b)} | + \frac {4}{P - 1} \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} \sum_ {i ^ {\prime} \neq i} | \overline {{\rho}} _ {j, r, i ^ {\prime}} ^ {(t, b)} | \\ \leq \frac {\mathrm {S N R}}{P - 1} \sqrt {\frac {8 C ^ {2} \log (6 n / \delta)}{d}} n \alpha + \frac {4}{P - 1} \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} n \alpha \\ \leq \frac {5}{P - 1} \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} n \alpha , \\ \end{array}
$$
where the second inequality is by (27) and (29), and the third inequality is by  $\mathrm{SNR} \leq 1 / \sqrt{8C'^2}$ . Therefore, for  $j = y_{i}$ , we have
$$
\left| \left\langle \mathbf {w} _ {j, r} ^ {(t, b)} - \mathbf {w} _ {j, r} ^ {(0, 0)}, \boldsymbol {\xi} _ {i} \right\rangle - \frac {1}{P - 1} \bar {\rho} _ {j, r, i} ^ {(t, b)} \right| \leq \frac {5}{P - 1} \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} n \alpha .
$$
Lemma C.4. Under Condition 3.1, suppose (27), (28) and (29) hold after  $b$ -th batch of  $t$ -th epoch. Then, for all  $j \neq y_i$ ,  $j \in \{\pm 1\}$  and  $i \in [n]$ ,  $F_j(\mathbf{W}_j^{(t,b)}, \mathbf{x}_i) \leq 0.5$ .
Proof of Lemma C.4. According to Lemma C.3, we have
$$
\begin{array}{l} F _ {j} \left(\mathbf {W} _ {j} ^ {(t, b)}, \mathbf {x} _ {i}\right) = \frac {1}{m} \sum_ {r = 1} ^ {m} \left[ \sigma \left(\left\langle \mathbf {w} _ {j, r} ^ {(t, b)}, y _ {i} \boldsymbol {\mu} \right\rangle\right) + (P - 1) \sigma \left(\left\langle \mathbf {w} _ {j, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \right\rangle\right) \right] \\ \leq 2 \max  \left\{\langle \mathbf {w} _ {j, r} ^ {(t, b)}, y _ {i} \boldsymbol {\mu} \rangle , (P - 1) \langle \mathbf {w} _ {j, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \rangle , 0 \right\} \\ \leq 6 \max  \left\{\langle \mathbf {w} _ {j, r} ^ {(0)}, y _ {i} \boldsymbol {\mu} \rangle , (P - 1) \langle \mathbf {w} _ {j, r} ^ {(0)}, \boldsymbol {\xi} _ {i} \rangle , \mathrm {S N R} \sqrt {\frac {3 2 \log (6 n / \delta)}{d}} n \alpha , y _ {i} j \gamma_ {j, r} ^ {(t, b)} \right., \\ \left. 5 \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} n \alpha + \underline {{\rho}} _ {j, r, i} ^ {(t, b)} \right\} \\ \leq 6 \max  \left\{\beta / 2, \operatorname {S N R} \sqrt {\frac {3 2 \log (6 n / \delta)}{d}} n \alpha , - \gamma_ {j, r} ^ {(t, b)}, 5 \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} n \alpha \right\} \\ \leq 0. 5, \\ \end{array}
$$
where the second inequality is by (30), (31) and (32); the third inequality is due to the definition of  $\beta$  and  $\underline{\rho}_{j,r,i}^{(t,b)} < 0$ ; the third inequality is by (25) and  $-\gamma_{j,r}^{(t,b)} \leq \frac{1}{12}$ .
Lemma C.5. Under Condition 3.1, suppose (27), (28) and (29) hold at  $b$ -th batch of  $t$ -th epoch. Then, it holds that
$$
\begin{array}{l} (P - 1) \langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \rangle \geq - 0. 2 5, \\ (P - 1) \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \right\rangle \leq (P - 1) \sigma \left(\left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \right\rangle\right) \leq (P - 1) \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \right\rangle + 0. 2 5, \\ \end{array}
$$
for any  $i \in [n]$ .
Proof of Lemma C.5. According to (32) in Lemma C.3, we have
$$
\begin{array}{l} (P - 1) \langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \pmb {\xi} _ {i} \rangle \geq (P - 1) \langle \mathbf {w} _ {y _ {i}, r} ^ {(0, 0)}, \pmb {\xi} _ {i} \rangle + \overline {{\rho}} _ {y _ {i}, r, i} ^ {(t, b)} - 5 n \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} \alpha \\ \geq - \beta - 5 n \sqrt {\frac {\log (6 n ^ {2} / \delta)}{d}} \alpha \\ \geq - 0. 2 5, \\ \end{array}
$$
where the second inequality is due to  $\overline{\rho}_{y_i,r,i}^{(t,b)}\geq 0$ , the third inequality is due to  $\beta < 1 / 8$  and  $5n\sqrt{\log(6n^2 / \delta) / d}\cdot \alpha < 1 / 8$  by Condition 3.1.
For the second equation, the first inequality holds naturally since  $z \leq \sigma(z)$ . For the inequality, if  $\langle \mathbf{w}_{y_i,r}^{(t)}, \pmb{\xi}_i \rangle \leq 0$ , we have
$$
(P - 1) \sigma (\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \rangle) = 0 \leq (P - 1) \langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \rangle + 0. 2 5.
$$
And if  $\langle \mathbf{w}_{y_i,r}^{(t,b)},\pmb {\xi}_i\rangle >0$  we have
$$
(P - 1) \sigma \left(\left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \right\rangle\right) = (P - 1) \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \right\rangle <   (P - 1) \left\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \right\rangle + 0. 2 5.
$$
Lemma C.6 (Lemma C.6 in Kou et al. (2023)). Let  $g(z) = \ell'(z) = -1 / (1 + \exp(z))$ , then for all  $z_2 - c \geq z_1 \geq -1$  where  $c \geq 0$  we have that
$$
\frac {\exp (c)}{4} \leq \frac {g (z _ {1})}{g (z _ {2})} \leq \exp (c).
$$
Lemma C.7. For any iteration  $t \in [0, T^{*})$  and  $b, b_{1}, b_{2} \in [\overline{H}]$ , we have the following statements hold:
1.  $\left|\sum_{r = 1}^{m}\left[\overline{\rho}_{y_i,r,i}^{(t,0)} - \overline{\rho}_{y_k,r,k}^{(t,0)}\right] - \sum_{r = 1}^{m}\left[\overline{\rho}_{y_i,r,i}^{(t,b_1)} - \overline{\rho}_{y_k,r,k}^{(t,b_2)}\right]\right| \leq 0.1\kappa.$
2.  $\langle \mathbf{w}_{y_i,r}^{(t,b)},\pmb {\xi}_i\rangle \geq \langle \mathbf{w}_{y_i,r}^{(t,0)},\pmb {\xi}_i\rangle -\sigma_0\sigma_p\sqrt{d} /\sqrt{2}$
3. Let  $\widetilde{S}_i^{(t,b)} = \{r\in [m]:\langle \mathbf{w}_{y_i,r}^{(t,b)},\pmb {\xi}_i\rangle >0\}$ , then we have
$$
S _ {i} ^ {(t, 0)} \subseteq \widetilde {S} _ {i} ^ {(t, b)}.
$$
4. Let  $\widetilde{S}_{j,r}^{(t,b)} = \{i\in [n]:y_i = j,\langle \mathbf{w}_{j,r}^{(t,b)},\pmb {\xi}_i\rangle >0\}$ , then we have
$$
S _ {j, r} ^ {(t, 0)} \subseteq \widetilde {S} _ {j, r} ^ {(t, b)}.
$$
Proof. For the first statement,
$$
\begin{array}{l} \left| \sum_ {r = 1} ^ {m} \left[ \bar {\rho} _ {y _ {i}, r, i} ^ {(t, 0)} - \bar {\rho} _ {y _ {k}, r, k} ^ {(t, 0)} \right] - \sum_ {r = 1} ^ {m} \left[ \bar {\rho} _ {y _ {i}, r, i} ^ {(t, b _ {1})} - \bar {\rho} _ {y _ {k}, r, k} ^ {(t, b _ {2})} \right] \right| \\ \leq \frac {\eta (P - 1) ^ {2}}{B m} \max  \left\{| S _ {i} ^ {(\widetilde {t} - 1, b _ {1})} | | \ell_ {i} ^ {\prime (\widetilde {t} - 1, b _ {1})} | \cdot \| \boldsymbol {\xi} _ {i} \| _ {2} ^ {2}, | S _ {k} ^ {(\widetilde {t} - 1, b _ {2})} | | \ell_ {k} ^ {\prime (\widetilde {t} - 1, b _ {2})} | \cdot \| \boldsymbol {\xi} _ {k} \| _ {2} ^ {2} \right\} \\ \leq \frac {\eta (P - 1) ^ {2}}{B} \frac {3 \sigma_ {p} ^ {2} d}{2} \\ \leq 0. 1 \kappa , \\ \end{array}
$$
where the first inequality follows from the iterative update rule of  $\overline{\rho}_{j,r,i}^{(t,b)}$ , the second inequality is due to Lemma B.2, and the last inequality is due to Condition 3.1.
For the second statement, recall that the stochastic gradient update rule is
$$
\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \rangle = \langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b - 1)}, \boldsymbol {\xi} _ {i} \rangle - \frac {\eta}{B m} \cdot \sum_ {i ^ {\prime} \in \mathcal {I} _ {t, b - 1}} \ell_ {i ^ {\prime}} ^ {(t, b - 1)} \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b - 1)}, y _ {i ^ {\prime}} \boldsymbol {\mu} \rangle) \cdot \langle y _ {i ^ {\prime}} \boldsymbol {\mu}, \boldsymbol {\xi} _ {i} \rangle y _ {i ^ {\prime}}
$$
$$
- \frac {\eta (P - 1)}{B m} \cdot \sum_ {i ^ {\prime} \in \mathcal {I} _ {t, b - 1} / i} \ell_ {i ^ {\prime}} ^ {(t, b - 1)} \cdot \sigma^ {\prime} (\langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b - 1)}, \boldsymbol {\xi} _ {i ^ {\prime}} \rangle) \cdot \langle \boldsymbol {\xi} _ {i ^ {\prime}}, \boldsymbol {\xi} _ {i} \rangle .
$$
$$
\begin{array}{l} \langle \mathbf {w} _ {y _ {i}, r} ^ {(t, b)}, \pmb {\xi} _ {i} \rangle \geq \langle \mathbf {w} _ {y _ {i}, r} ^ {(t, 0)}, \pmb {\xi} _ {i} \rangle - \frac {\eta}{B m} \cdot n \cdot \| \mu \| _ {2} \sigma_ {p} \sqrt {2 \log (6 n / \delta)} - \frac {\eta (P - 1)}{B m} \cdot n \cdot 2 \sigma_ {p} ^ {2} \sqrt {d \log (6 n ^ {2} / \delta)} \\ \geq \langle \mathbf {w} _ {y _ {i}, r} ^ {(t, 0)}, \pmb {\xi} _ {i} \rangle - \sigma_ {0} \sigma_ {p} \sqrt {d} / \sqrt {2}, \\ \end{array}
$$
where the first inequality is due to Lemma B.1, and the second inequality is due to Condition 3.1.
For the third statement. Let  $r^* \in S_i^{(t,0)}$ , then
$$
\langle \mathbf {w} _ {y _ {i}, r ^ {*}} ^ {(t, b)}, \boldsymbol {\xi} _ {i} \rangle \geq \langle \mathbf {w} _ {y _ {i}, r ^ {*}} ^ {(t, 0)}, \boldsymbol {\xi} _ {i} \rangle - \sigma_ {0} \sigma_ {p} \sqrt {d} / \sqrt {2} > 0,
$$
where the first inequality is due to the second statement, and the second inequality is due to the definition of  $\widetilde{S}_i^{t,0}$ . Therefore,  $r^* \in \widetilde{S}_i^{(t,b)}$  and  $S_i^{(t,b)} \subseteq \widetilde{S}_i^{(t,b)}$ . The fourth statement can be obtained similarly.
Lemma C.8. Under Assumption 3.1, suppose (27), (28) and (29) hold for any iteration  $(t', b') \leq (t, 0)$ . Then, the following conditions also hold for  $\forall t' \leq t$  and  $\forall b', b_1', b_2' \in [H]$ :
1.  $\sum_{r=1}^{m}\left[\overline{\rho}_{y_i,r,i}^{(t',0)} - \overline{\rho}_{y_k,r,k}^{(t',0)}\right] \leq \kappa$  for all  $i, k \in [n]$ .
2.  $y_{i} \cdot f(\mathbf{W}^{(t^{\prime}, b_{1}^{\prime})}, \mathbf{x}_{i}) - y_{k} \cdot f(\mathbf{W}^{(t^{\prime}, b_{2}^{\prime})}, \mathbf{x}_{k}) \leq C_{1}$  for all  $i, k \in [n]$ .
3.  $\ell_i^{\prime (t',b_1')} / \ell_k^{\prime (t',b_2')}\leq C_2 = \exp (C_1)$  for all  $i,k\in [n]$
4.  $S_{i}^{(0,0)} \subseteq S_{i}^{(t',0)}$ , where  $S_{i}^{(t',0)} := \{r \in [m] : \langle \mathbf{w}_{y_{i},r}^{(t',0)}, \pmb{\xi}_{i} \rangle > \sigma_{0} \sigma_{p} \sqrt{d} / \sqrt{2}\}$ , and hence  $|S_{i}^{(t',0)}| \geq 0.8 m \Phi(-1)$  for all  $i \in [n]$ .
5.  $S_{j,r}^{(0,0)} \subseteq S_{j,r}^{(t',0)}$ , where  $S_{j,r}^{(t',0)} := \{i \in [n] : y_i = j, \langle \mathbf{w}_{j,r}^{(t',0)}, \pmb{\xi}_i \rangle > \sigma_0 \sigma_p \sqrt{d} / \sqrt{2}\}$ , and hence  $|S_{j,r}^{(t',0)}| \geq \Phi(-1) n / 4$  for all  $j \in \{\pm 1\}, r \in [m]$ .
Here we take  $\kappa$  and  $C_1$  as 10 and 5 respectively.
Proof of Lemma C.8. We prove Lemma C.8 by induction. When  $t' = 0$ , the fourth and fifth conditions hold naturally by Lemma B.3 and B.4.
For the first condition, since we have  $\overline{\rho}_{j,r,i}^{(0,0)} = 0$  for any  $j,r,i$  according to (26), it is straightforward that  $\sum_{r=1}^{m}\left[\overline{\rho}_{y_i,r,i}^{(0,0)} - \overline{\rho}_{y_k,r,k}^{(0,0)}\right] = 0$  for all  $i,k \in [n]$ . So the first condition holds for  $t' = 0$ .
For the second condition, we have
$$
\begin{array}{l} y _ {i} \cdot f (\mathbf {W} ^ {(0, 0)}, \mathbf {x} _ {i}) - y _ {k} \cdot f (\mathbf {W} ^ {(0, 0)}, \mathbf {x} _ {k}) \\ = F _ {y _ {i}} (\mathbf {W} _ {y _ {i}} ^ {(0, 0)}, \mathbf {x} _ {i}) - F _ {- y _ {i}} (\mathbf {W} _ {- y _ {i}} ^ {(0, 0)}, \mathbf {x} _ {i}) + F _ {- y _ {k}} (\mathbf {W} _ {- y _ {k}} ^ {(0, 0)}, \mathbf {x} _ {i}) - F _ {y _ {k}} (\mathbf {W} _ {y _ {k}} ^ {(0, 0)}, \mathbf {x} _ {i}) \\ \leq F _ {y _ {i}} \left(\mathbf {W} _ {y _ {i}} ^ {(0, 0)}, \mathbf {x} _ {i}\right) + F _ {- y _ {k}} \left(\mathbf {W} _ {- y _ {k}} ^ {(0, 0)}, \mathbf {x} _ {i}\right) \\ = \frac {1}{m} \sum_ {r = 1} ^ {m} \left[ \sigma \left(\left\langle \mathbf {w} _ {y _ {i}, r} ^ {(0, 0)}, y _ {i} \boldsymbol {\mu} \right\rangle\right) + (P - 1) \sigma \left(\left\langle \mathbf {w} _ {y _ {i}, r} ^ {(0, 0)}, \boldsymbol {\xi} _ {i} \right\rangle\right) \right] \\ + \frac {1}{m} \sum_ {r = 1} ^ {m} [ \sigma (\langle \mathbf {w} _ {- y _ {k}, r} ^ {(0, 0)}, y _ {k} \boldsymbol {\mu} \rangle) + (P - 1) \sigma (\langle \mathbf {w} _ {- y _ {k}, r} ^ {(0, 0)}, \boldsymbol {\xi} _ {i} \rangle) ] \\ \leq 4 \beta \leq 1 / 3 \leq C _ {1}, \\ \end{array}
$$
where the first inequality is by  $F_{j}(\mathbf{W}_{j}^{(0,0)},\mathbf{x}_{i}) > 0$ , the second inequality is due to (23), and the third inequality is due to (25).
By Lemma C.6 and the second condition, the third condition can be obtained directly as
$$
\frac {\ell_ {i} ^ {\prime (0 , 0)}}{\ell_ {k} ^ {\prime (0 , 0)}} \leq \exp \left(y _ {k} \cdot f (\mathbf {W} ^ {(0, 0)}, \mathbf {x} _ {k}) - y _ {i} \cdot f (\mathbf {W} ^ {(0, 0)}, \mathbf {x} _ {i})\right) \leq \exp (C _ {1}).
$$
Now suppose there exists  $(\widetilde{t},\widetilde{b})\leq (t,b)$  such that these five conditions hold for any  $(0,0)\leq (t^{\prime},b^{\prime}) < (\widetilde{t},\widetilde{b})$ . We aim to prove that these conditions also hold for  $(t^{\prime},b^{\prime}) = (\widetilde{t},\widetilde{b})$ .
We first show that, for any  $0 \leq t' \leq t$  and  $0 \leq b_1', b_2' \leq b$ ,  $y_i \cdot f(\mathbf{W}^{(t', b_1')}, \mathbf{x}_i) - y_k \cdot f(\mathbf{W}^{(t', b_2')}, \mathbf{x}_k)$  can be approximated by  $\frac{1}{m} \sum_{r=1}^{m} \left[ \overline{\rho}_{y_i, r, i}^{(t', b_1')} - \overline{\rho}_{y_k, r, k}^{(t', b_2')} \right]$  with a small constant approximation error. We begin by writing out