SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \mathbb {E} \left\| \frac {1}{n} \sum_ {i = 1} ^ {n} \nabla f (\theta , \hat {x} _ {i}) - \frac {1}{b} \sum_ {i \in \Omega_ {1}} \nabla f (\theta , \hat {x} _ {i}) \right\| ^ {2} = \mathbb {E} \left\| \frac {1}{b} \sum_ {i \in \Omega_ {1}} \left(\frac {1}{n} \sum_ {j = 1} ^ {n} \nabla f (\theta , \hat {x} _ {j}) - \nabla f (\theta , \hat {x} _ {i})\right) \right\| ^ {2} \\ = \mathbb {E} \left(\frac {1}{b} \sum_ {i \in \Omega_ {1}} K _ {2} \frac {1}{n} \sum_ {j = 1} ^ {n} \| \hat {x} _ {i} - \hat {x} _ {j} \| (2 \| \theta \| + 1)\right) ^ {2} \\ \leq 4 D ^ {2} K _ {2} ^ {- 2} (2 \| \theta \| + 1) ^ {2} \\ \leq 8 D ^ {2} K _ {2} ^ {2} \left(4 \| \theta \| ^ {2} + 1\right) \\ \leq 8 D ^ {2} K _ {2} ^ {2} \left(8 \| \theta - \hat {\theta} _ {*} \| ^ {2} + 8 \| \hat {\theta} _ {*} \| ^ {2} + 1\right). \tag {D.15} \\ \end{array}
$$
$$
\begin{array}{l} (\hat {P} \hat {V}) (\theta) \leq \left(1 - 2 \eta \mu + \eta^ {2} K _ {1} ^ {2} + 6 4 \eta^ {2} D ^ {2} K _ {2} ^ {2}\right) \| \theta - \hat {\theta} _ {*} \| ^ {2} + 1 + 8 \eta^ {2} D ^ {2} K _ {2} ^ {2} \left(8 \| \hat {\theta} _ {*} \| ^ {2} + 1\right) \\ = \left(1 - 2 \eta \mu + \eta^ {2} K _ {1} ^ {2} + 6 4 \eta^ {2} D ^ {2} K _ {2} ^ {2}\right) \hat {V} (\theta) \\ + 2 \eta \mu - \eta^ {2} K _ {1} ^ {2} - 5 6 \eta^ {2} D ^ {2} K _ {2} ^ {2} + 6 4 \eta^ {2} D ^ {2} K _ {2} ^ {2} \| \hat {\theta} _ {*} \| ^ {2} \\ \leq (1 - \eta \mu) \hat {V} (\theta) + 2 \eta \mu - \eta^ {2} K _ {1} ^ {2} - 5 6 \eta^ {2} D ^ {2} K _ {2} ^ {2} + 6 4 \eta^ {2} D ^ {2} K _ {2} ^ {2} \| \hat {\theta} _ {*} \| ^ {2}, \tag {D.16} \\ \end{array}
$$
provided that  $\eta \leq \frac{\mu}{K_1^2 + 64D^2K_2^2}$ . This completes the proof.
Next, we estimate the perturbation gap based on the Lyapunov function  $\hat{V}$ .
Lemma D.3. Assume that Assumption 3.1 holds. Assume that  $\sup_{x\in \mathcal{X}}\| x\| \leq D$  for some  $D < \infty$ . Then, we have
$$
\sup  _ {\theta \in \mathbb {R} ^ {d}} \frac {\mathcal {W} _ {1} \left(\delta_ {\theta} P , \delta_ {\theta} \hat {P}\right)}{\hat {V} (\theta)} \leq \frac {4 D K _ {2} \eta}{n} \left(2 \| \hat {\theta} _ {*} \| + 1\right), \tag {D.17}
$$
where  $\hat{\theta}_{*}$  is the minimizer of  $\hat{F} (\theta ,\hat{X}_n)\coloneqq \frac{1}{n}\sum_{i = 1}^{n}\nabla f(\theta ,\hat{x}_i)$ .
$$
\theta_ {1} = \theta - \frac {\eta}{b} \sum_ {i \in \Omega_ {1}} \nabla f \left(\theta , x _ {i}\right), \tag {D.18}
$$
$$
\hat {\theta} _ {1} = \theta - \frac {\eta}{b} \sum_ {i \in \Omega_ {1}} \nabla f \left(\theta , \hat {x} _ {i}\right), \tag {D.19}
$$
$$
\begin{array}{l} \mathcal {W} _ {1} \left(\delta_ {\theta} P, \delta_ {\theta} \hat {P}\right) \leq \frac {\eta}{b} \mathbb {E} \left\| \sum_ {i \in \Omega_ {1}} \nabla f (\theta , x _ {i}) - \nabla f (\theta , \hat {x} _ {i}) \right\| \\ \leq \frac {\eta}{b} \mathbb {E} \left[ \sum_ {i \in \Omega_ {1}} \| \nabla f (\theta , x _ {i}) - \nabla f (\theta , \hat {x} _ {i}) \| \right] \\ \leq \frac {\eta}{b} \mathbb {E} \left[ \sum_ {i \in \Omega_ {1}} K _ {2} \| x _ {i} - \hat {x} _ {i} \| (2 \| \theta \| + 1) \right] \tag {D.20} \\ \end{array}
$$
Since  $X_{n}$  and  $\hat{X}_{n}$  differ by at most one element and  $\sup_{x\in \mathcal{X}}\| x\| \leq D$  for some  $D < \infty$ , we have  $x_{i} = \hat{x}_{i}$  for any  $i\in \Omega_1$  with probability  $\frac{n - b}{n}$  and  $x_{i}\neq \hat{x}_{i}$  for exactly one  $i\in \Omega_1$  with probability  $\frac{b}{n}$  and therefore
$$
\mathcal {W} _ {1} \left(\delta_ {\theta} P, \delta_ {\theta} \hat {P}\right) \leq \frac {\eta}{b} \frac {b}{n} 2 K _ {2} D (2 \| \theta \| + 1) = \frac {2 D K _ {2} \eta}{n} (2 \| \theta \| + 1). \tag {D.21}
$$
$$
\begin{array}{l} \sup _ {\theta \in \mathbb {R} ^ {d}} \frac {\mathcal {W} _ {1} (\delta_ {\theta} P , \delta_ {\theta} \hat {P})}{\hat {V} (\theta)} \leq \sup _ {\theta \in \mathbb {R} ^ {d}} \frac {4 D K _ {2} \eta}{n} \frac {\| \theta \| + \frac {1}{2}}{1 + \| \theta - \hat {\theta} _ {*} \| ^ {2}} \\ \leq \sup  _ {\theta \in \mathbb {R} ^ {d}} \frac {4 D K _ {2} \eta}{n} \frac {2 \| \theta - \hat {\theta} _ {*} \| + 2 \| \hat {\theta} _ {*} \| + 1}{1 + \| \theta - \hat {\theta} _ {*} \| ^ {2}} \\ \leq \sup  _ {\theta \in \mathbb {R} ^ {d}} \frac {4 D K _ {2} \eta}{n} \frac {2 \| \theta - \hat {\theta} _ {*} \| (2 \| \hat {\theta} _ {*} \| + 1)}{1 + \| \theta - \hat {\theta} _ {*} \| ^ {2}} \\ \leq \frac {4 D K _ {2} \eta}{n} \left(2 \| \hat {\theta} _ {*} \| + 1\right). \tag {D.22} \\ \end{array}
$$
Next, let us provide a technical lemma that upper bounds the norm of  $\theta_*$  and  $\hat{\theta}_*$ , which are the minimizers of  $\hat{F}(\theta, X_n) \coloneqq \frac{1}{n} \sum_{i=1}^{n} \nabla f(\theta, x_i)$  and  $\hat{F}(\theta, \hat{X}_n) \coloneqq \frac{1}{n} \sum_{i=1}^{n} \nabla f(\theta, \hat{x}_i)$  respectively.
Lemma D.4. Under Assumption 3.2, we have  $\| \theta_{*}\| \leq \frac{1}{\mu}\sup_{x\in \mathcal{X}}\| \nabla f(0,x)\|$  and  $\| \hat{\theta}_*\| \leq \frac{1}{\mu}\sup_{x\in \mathcal{X}}\| \nabla f(0,x)\|$ .
Proof. Since  $f(\theta, x)$  is  $\mu$ -strongly convex in  $\theta$  for every  $x \in \mathcal{X}$ , we have
$$
\begin{array}{l} \left\langle \nabla \hat {F} (0, X _ {n}) - \nabla \hat {F} (\theta_ {*}, X _ {n}), 0 - \theta_ {*} \right\rangle = - \frac {1}{n} \sum_ {i = 1} ^ {n} \langle \nabla f (0, x _ {i}), \theta_ {*} \rangle \\ = \frac {1}{n} \sum_ {i = 1} ^ {n} \left\langle \nabla f \left(0, x _ {i}\right) - \nabla f \left(\theta_ {*}, x _ {i}\right), 0 - \theta_ {*} \right\rangle \geq \mu \| \theta_ {*} \| ^ {2}, \tag {D.23} \\ \end{array}
$$
$$
\mu \| \theta_ {*} \| ^ {2} \leq \sup  _ {x \in \mathcal {X}} \| \nabla f (0, x) \| \cdot \| \theta_ {*} \|, \tag {D.24}
$$
which yields that  $\| \theta_{*}\| \leq \frac{1}{\mu}\sup_{x\in \mathcal{X}}\| \nabla f(0,x)\|$ . Similarly, one can show that  $\| \hat{\theta}_{*}\| \leq \frac{1}{\mu}\sup_{x\in \mathcal{X}}\| \nabla f(0,x)\|$ . This completes the proof.
Proof. By applying Lemma D.1, Lemma D.2, Lemma D.3 and Lemma 2.1, we obtain
$$
\begin{array}{l} \mathcal {W} _ {1} (\nu_ {k}, \hat {\nu} _ {k}) \leq \frac {8 D K _ {2} (1 - \left(1 - \frac {\eta \mu}{2}\right) ^ {k})}{n \mu} \left(2 \| \hat {\theta} _ {*} \| + 1\right) \\ \cdot \max  \left\{1 + \left\| \theta - \hat {\theta} _ {*} \right\| ^ {2}, 2 - \frac {\eta}{\mu} K _ {1} ^ {2} - \frac {5 6 \eta}{\mu} D ^ {2} K _ {2} ^ {2} + \frac {6 4 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \| \hat {\theta} _ {*} \| ^ {2} \right\}, \tag {D.25} \\ \end{array}
$$
where  $\hat{\theta}_{*}$  is the minimizer of  $\hat{F} (\theta ,\hat{X}_n)\coloneqq \frac{1}{n}\sum_{i = 1}^{n}\nabla f(\theta ,\hat{x}_i)$ . Finally,  $\| \theta -\hat{\theta}_{*}\|^{2}\leq 2\| \theta \|^{2} + 2\| \hat{\theta}_{*}\|^{2}$  and by applying Lemma D.4, we complete the proof.
E Proofs of Wasserstein Perturbation Bounds: Non-Convex Case
Lemma E.1. Assume that Assumption 3.1, Assumption 3.3 and Assumption 3.4 hold. For any  $\hat{\eta} \in (0,1)$ . Define  $M > 0$  so that  $\int_{\| \theta_1 \| \leq M} p(\theta_* , \theta_1) d\theta_1 \geq \sqrt{\hat{\eta}}$  and any  $R > 0$  so that  $R > \frac{2K_0}{m}$  and
$$
\inf  _ {\theta , \theta_ {1} \in \mathbb {R} ^ {d}: V (\theta) \leq R, \| \theta_ {1} \| \leq M} \frac {p \left(\theta , \theta_ {1}\right)}{p \left(\theta_ {*}, \theta_ {1}\right)} \geq \sqrt {\hat {\eta}}, \tag {E.1}
$$
$$
K _ {0} := 2 m - \eta K _ {1} ^ {2} - 5 6 \eta D ^ {2} K _ {2} ^ {2} + 6 4 \eta D ^ {2} K _ {2} ^ {2} \| \theta_ {*} \| ^ {2} + 2 K + \eta \sigma^ {2}. \tag {E.2}
$$
Then, for any  $n\in \mathbb{N}$
$$
\mathcal {W} _ {1} \left(P ^ {n} (\theta , \cdot), P ^ {n} (\tilde {\theta}, \cdot)\right) \leq \frac {1}{2 \sqrt {\psi (1 + \psi)}} \bar {\eta} ^ {n} d _ {\psi} (\delta_ {\theta}, \delta_ {\tilde {\theta}}), \tag {E.3}
$$
for any  $\theta, \tilde{\theta}$  in  $\mathbb{R}^d$ , where for any  $\eta_0 \in (0, \hat{\eta})$  and  $\gamma_0 \in \left(1 - m\eta + \frac{2\eta K_0}{R}, 1\right)$  one can choose  $\psi = \frac{\eta_0}{\eta K_0}$  and  $\bar{\eta} = (1 - (\hat{\eta} - \eta_0)) \vee \frac{2 + R\psi\gamma_0}{2 + R\psi}$  and  $d_{\psi}$  is the weighted total variation distance defined in Section B.1.
Proof. Our proof relies on a computable bound on the Wasserstein contraction for the Markov chains by [HM11] that satisfies a drift condition (Assumption B.1) that relies on the construction of an appropriate Lyapunov function and a minorization condition (Assumption B.2).
By applying Lemma E.3, we can immediately show that the following drift condition holds. Let  $V(\theta) \coloneqq 1 + \| \theta - \theta_* \|^2$ , where  $\theta_*$  is the minimizer of  $\frac{1}{n} \sum_{i=1}^{n} \nabla f(\theta, x_i)$ . Assume that  $\eta < \min \left\{ \frac{1}{m}, \frac{m}{K_1 + 64D^2 K_2^2} \right\}$  and  $\sup_{x \in \mathcal{X}} \| x \| \leq D$  for some  $D < \infty$ . Then, we have
$$
(P V) (\theta) \leq (1 - m \eta) V (\theta) + \eta K _ {0}, \tag {E.4}
$$
$$
K _ {0} := 2 m - \eta K _ {1} ^ {2} - 5 6 \eta D ^ {2} K _ {2} ^ {2} + 6 4 \eta D ^ {2} K _ {2} ^ {2} \| \theta_ {*} \| ^ {2} + 2 K + \eta \sigma^ {2}. \tag {E.5}
$$
Thus, the drift condition (Assumption B.1) holds.
Next, let us show that the minorization condition (Assumption B.2) also holds. Let us recall that
$$
\theta_ {1} = \theta - \frac {\eta}{b} \sum_ {i \in \Omega_ {1}} \nabla f \left(\theta , x _ {i}\right) + \eta \xi_ {1}. \tag {E.6}
$$
We denote  $p(\theta, \theta_1)$  the probability density function of  $\theta_1$  with the emphasis on the dependence on the initial point  $\theta$ . Then, to check that the minorization condition (Assumption B.2) holds, it suffices to show that there exists some constant  $\hat{\eta} \in (0, 1)$
$$
\inf  _ {\theta \in \mathbb {R} ^ {d}: V (\theta) \leq R} p \left(\theta , \theta_ {1}\right) \geq \hat {\eta} q \left(\theta_ {1}\right), \quad \text {f o r a n y} \theta_ {1} \in \mathbb {R} ^ {d}, \tag {E.7}
$$
for some  $R > \frac{2\eta K_0}{1 - (1 - m\eta)} = \frac{2K_0}{m}$ , where  $q(\theta_1)$  is the density function of a probability distribution function on  $\mathbb{R}^d$ , and (E.7) follows from Lemma E.2. Hence, by Lemma B.1, we have
$$
d _ {\psi} \left(P ^ {n} (\theta , \cdot), P ^ {n} (\tilde {\theta}, \cdot)\right) \leq \bar {\eta} ^ {n} d _ {\psi} (\delta_ {\theta}, \delta_ {\tilde {\theta}}),
$$
for any  $\theta, \tilde{\theta}$  in  $\mathbb{R}^d$ , where for any  $\eta_0 \in (0, \hat{\eta})$  and  $\gamma_0 \in \left(1 - m\eta + \frac{2\eta K_0}{R}, 1\right)$  one can choose  $\psi = \frac{\eta_0}{\eta K_0}$  and  $\bar{\eta} = (1 - (\hat{\eta} - \eta_0)) \vee \frac{2 + R\psi\gamma_0}{2 + R\psi}$ .
Finally, by the Kantorovich-Rubinstein duality for the Wasserstein metric, we get for any two probability measures  $\mu_1,\mu_2$  on  $\mathbb{R}^d$ :
$$
\begin{array}{l} \mathcal {W} _ {1} \left(\mu_ {1}, \mu_ {2}\right) = \sup  \left\{\int_ {\mathbb {R} ^ {d}} \phi (\theta) \left(\mu_ {1} - \mu_ {2}\right) (d \theta): \phi \text {i s} 1 \text {- L i p s c h i t z} \right\} \\ = \sup  \left\{\int_ {\mathbb {R} ^ {d}} \left(\phi (\theta) - \phi \left(\theta_ {*}\right)\right) \left(\mu_ {1} - \mu_ {2}\right) (d \theta): \phi \text {i s} 1 \text {- L i p s c h i t z} \right\} \\ \leq \int_ {\mathbb {R} ^ {d}} \| \theta - \theta_ {*} \| | \mu_ {1} - \mu_ {2} | (d \theta) \\ \leq \frac {1}{2 \sqrt {\psi (1 + \psi)}} \int_ {\mathbb {R} ^ {d}} (1 + \psi V (\theta)) | \mu_ {1} - \mu_ {2} | (d \theta) \\ = \frac {1}{2 \sqrt {\psi (1 + \psi)}} d _ {\psi} \left(\mu_ {1}, \mu_ {2}\right). \tag {E.8} \\ \end{array}
$$
$$
\mathcal {W} _ {1} \left(P ^ {n} (\theta , \cdot), P ^ {n} (\tilde {\theta}, \cdot)\right) \leq \frac {1}{2 \sqrt {\psi (1 + \psi)}} \bar {\eta} ^ {n} d _ {\psi} (\delta_ {\theta}, \delta_ {\tilde {\theta}}).
$$
The proof of Lemma E.1 relies on the following technical lemma, which is a reformulation of Lemma 35 in [CGZ19] that helps establish the minorization condition (Assumption B.2).
Lemma E.2. For any  $\hat{\eta} \in (0,1)$  and  $M > 0$  so that  $\int_{\|\theta_1 - \theta_*\| \leq M} p(\theta_*,\theta_1)d\theta_1 \geq \sqrt{\hat{\eta}}$  and any  $R > 0$  so that
$$
\inf  _ {\theta , \theta_ {1} \in \mathbb {R} ^ {d}: V (\theta) \leq R, \| \theta_ {1} - \theta_ {*} \| \leq M} \frac {p \left(\theta , \theta_ {1}\right)}{p \left(\theta_ {*}, \theta_ {1}\right)} \geq \sqrt {\hat {\eta}}. \tag {E.9}
$$
$$
\inf  _ {\theta \in \mathbb {R} ^ {d}: V (\theta) \leq R} p (\theta , \theta_ {1}) \geq \hat {\eta} q (\theta_ {1}), \quad f o r a n y \theta_ {1} \in \mathbb {R} ^ {d}, \tag {E.10}
$$
$$
q \left(\theta_ {1}\right) = p \left(\theta_ {*}, \theta_ {1}\right) \cdot \frac {1 _ {\| \theta_ {1} - \theta_ {*} \| \leq M}}{\int_ {\| \theta_ {1} - \theta_ {*} \| \leq M} p \left(\theta_ {*}, \theta_ {1}\right) d \theta_ {1}}. \tag {E.11}
$$
Proof. The proof is an adaptation of the proof of Lemma 35 in [CGZ19]. Let us take:
$$
q \left(\theta_ {1}\right) = p \left(\theta_ {*}, \theta_ {1}\right) \cdot \frac {1 _ {\| \theta_ {1} - \theta_ {*} \| \leq M}}{\int_ {\| \theta_ {1} - \theta_ {*} \| \leq M} p \left(\theta_ {*}, \theta_ {1}\right) d \theta_ {1}}. \tag {E.12}
$$
Then, it is clear that  $q(\theta_1)$  is a probability density function on  $\mathbb{R}^d$ . It follows that (E.7) automatically holds for  $\| \theta_1 - \theta_* \| > M$ . Thus, we only need to show that (E.7) holds for  $\| \theta_1 - \theta_* \| \leq M$ . Since  $\xi_1$  has a continuous density,  $p(\theta, \theta_1)$  is continuous in both  $\theta$  and  $\theta_1$ . Fix  $M$ , by continuity of  $p(\theta, \theta_1)$  in both  $\theta$  and  $\theta_1$ , there exists some  $\eta' \in (0, 1)$  such that uniformly in  $\| \theta_1 - \theta_* \| \leq M$ ,
$$
\inf  _ {\theta \in \mathbb {R} ^ {d}: V (\theta) \leq R} p \left(\theta , \theta_ {1}\right) \geq \eta^ {\prime} p \left(\theta_ {*}, \theta_ {1}\right) = \hat {\eta} q \left(\theta_ {1}\right), \tag {E.13}
$$
where we can take
$$
\hat {\eta} := \eta^ {\prime} \int_ {\left\| \theta_ {1} - \theta_ {*} \right\| \leq M} p \left(\theta_ {*}, \theta_ {1}\right) d \theta_ {1}. \tag {E.14}
$$
In particular, for any fixed  $\hat{\eta}$ , we can take  $M > 0$  such that
$$
\int_ {\left\| \theta_ {1} - \theta_ {*} \right\| \leq M} p \left(\theta_ {*}, \theta_ {1}\right) d \theta_ {1} \geq \sqrt {\hat {\eta}}, \tag {E.15}
$$
and with fixed  $\eta$  and  $M$ , we take  $R > 0$  such that uniformly in  $\| \theta_1 - \theta_*\| \leq M$ ,
$$
\inf  _ {\theta \in \mathbb {R} ^ {d}: V (\theta) \leq R} p \left(\theta , \theta_ {1}\right) \geq \sqrt {\hat {\eta}} p \left(\theta_ {*}, \theta_ {1}\right). \tag {E.16}
$$
Lemma E.3. Assume that Assumption 3.1, Assumption 3.3 and Assumption 3.4 hold. Let  $\hat{V}(\theta) := 1 + \| \theta - \hat{\theta}_* \|^2$ , where  $\hat{\theta}_*$  is the minimizer of  $\hat{F}(\theta, \hat{X}_n) := \frac{1}{n} \sum_{i=1}^{n} \nabla f(\theta, \hat{x}_i)$ . Assume that  $\eta < \min \left\{\frac{1}{m}, \frac{m}{K_1^2 + 64D^2 K_2^2}\right\}$  and  $\sup_{x \in \mathcal{X}} \|x\| \leq D$  for some  $D < \infty$ . Then, we have
$$
(\hat {P} \hat {V}) (\theta) \leq (1 - m \eta) \hat {V} (\theta) + 2 m \eta - \eta^ {2} K _ {1} ^ {2} - 5 6 \eta^ {2} D ^ {2} K _ {2} ^ {2} + 6 4 \eta^ {2} D ^ {2} K _ {2} ^ {2} \| \hat {\theta} _ {*} \| ^ {2} + 2 \eta K + \eta^ {2} \sigma^ {2}. \tag {E.17}
$$
$$
\hat {\theta} _ {k} = \hat {\theta} _ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) + \eta \xi_ {k}. \tag {E.18}
$$
$$
\begin{array}{l} \hat {\theta} _ {1} = \theta - \frac {\eta}{b} \sum_ {i \in \Omega_ {1}} \nabla f (\theta , \hat {x} _ {i}) + \eta \xi_ {1} \\ = \theta - \frac {\eta}{n} \sum_ {i = 1} ^ {n} \nabla f \left(\theta , \hat {x} _ {i}\right) + \eta \left(\frac {1}{n} \sum_ {i = 1} ^ {n} \nabla f \left(\theta , \hat {x} _ {i}\right) - \frac {1}{b} \sum_ {i \in \Omega_ {1}} \nabla f \left(\theta , \hat {x} _ {i}\right)\right) + \eta \xi_ {1}. \tag {E.19} \\ \end{array}
$$
$$
\mathbb {E} \left[ \frac {1}{b} \sum_ {i \in \Omega_ {1}} \nabla f (\theta , \hat {x} _ {i}) | \theta \right] = \frac {1}{n} \sum_ {i = 1} ^ {n} \nabla f (\theta , \hat {x} _ {i}). \tag {E.20}
$$
$$
\begin{array}{l} (\hat {P} \hat {V}) (\theta) \\ = \mathbb {E} \hat {V} (\hat {\theta} _ {1}) = 1 + \mathbb {E} \left\| \hat {\theta} _ {1} - \hat {\theta} _ {*} \right\| ^ {2} \\ = 1 + \left\| \theta - \hat {\theta} _ {*} - \frac {\eta}{n} \sum_ {i = 1} ^ {n} \nabla f \left(\theta , \hat {x} _ {i}\right) \right\| ^ {2} + \eta^ {2} \mathbb {E} \left\| \frac {1}{n} \sum_ {i = 1} ^ {n} \nabla f \left(\theta , \hat {x} _ {i}\right) - \frac {1}{b} \sum_ {i \in \Omega_ {1}} \nabla f \left(\theta , \hat {x} _ {i}\right) \right\| ^ {2} + \eta^ {2} \sigma^ {2}. \tag {E.21} \\ \end{array}
$$
$$
\begin{array}{l} \left\| \theta - \hat {\theta} _ {*} - \frac {\eta}{n} \sum_ {i = 1} ^ {n} \nabla f (\theta , \hat {x} _ {i}) \right\| ^ {2} \\ = \left\| \theta - \hat {\theta} _ {*} - \frac {\eta}{n} \sum_ {i = 1} ^ {n} \left(\nabla f (\theta , \hat {x} _ {i}) - \nabla f (\hat {\theta} _ {*}, \hat {x} _ {i})\right) \right\| ^ {2} \\ \leq \left(1 - 2 m \eta\right) \left\| \theta - \hat {\theta} _ {*} \right\| ^ {2} + 2 \eta K + \frac {\eta^ {2}}{n ^ {2}} \left\| \sum_ {i = 1} ^ {n} \left(\nabla f (\theta , \hat {x} _ {i}) - \nabla f \left(\hat {\theta} _ {*}, \hat {x} _ {i}\right)\right) \right\| ^ {2} \\ \leq \left(1 - 2 m \eta + \eta^ {2} K _ {1} ^ {2}\right) \left\| \theta - \hat {\theta} _ {*} \right\| ^ {2} + 2 \eta K. \tag {E.22} \\ \end{array}
$$
Moreover, by following the same arguments as in the proof of Lemma D.2, we have
$$
\mathbb {E} \left\| \frac {1}{n} \sum_ {i = 1} ^ {n} \nabla f (\theta , \hat {x} _ {i}) - \frac {1}{b} \sum_ {i \in \Omega_ {1}} \nabla f (\theta , \hat {x} _ {i}) \right\| ^ {2} \leq 8 D ^ {2} K _ {2} ^ {2} \left(8 \| \theta - \hat {\theta} _ {*} \| ^ {2} + 8 \| \hat {\theta} _ {*} \| ^ {2} + 1\right). \tag {E.23}
$$
$$
\begin{array}{l} (\hat {P} \hat {V}) (\theta) \leq \left(1 - 2 m \eta + \eta^ {2} K _ {1} ^ {2} + 6 4 \eta^ {2} D ^ {2} K _ {2} ^ {2}\right) \left\| \theta - \hat {\theta} _ {*} \right\| ^ {2} \\ + 1 + 8 \eta^ {2} D ^ {2} K _ {2} ^ {2} \left(8 \| \hat {\theta} _ {*} \| ^ {2} + 1\right) + 2 \eta K + \eta^ {2} \sigma^ {2} \\ = \left(1 - 2 m \eta + \eta^ {2} K _ {1} ^ {- 2} + 6 4 \eta^ {2} D ^ {2} K _ {2} ^ {2}\right) \hat {V} (\theta) \\ + 2 m \eta - \eta^ {2} K _ {1} ^ {2} - 5 6 \eta^ {2} D ^ {2} K _ {2} ^ {2} + 6 4 \eta^ {2} D ^ {2} K _ {2} ^ {2} \| \hat {\theta} _ {*} \| ^ {2} + 2 \eta K + \eta^ {2} \sigma^ {2} \\ \leq (1 - m \eta) \hat {V} (\theta) + 2 m \eta - \eta^ {2} K _ {1} ^ {2} - 5 6 \eta^ {2} D ^ {2} K _ {2} ^ {2} + 6 4 \eta^ {2} D ^ {2} K _ {2} ^ {2} \| \hat {\theta} _ {*} \| ^ {2} + 2 \eta K + \eta^ {2} \sigma^ {2}, \tag {E.24} \\ \end{array}
$$
provided that  $\eta \leq \frac{m}{K_1^2 + 64D^2K_2^2}$ . This completes the proof.
Lemma E.4. Assume that Assumption 3.1 and Assumption 3.4 hold. Assume that  $\sup_{x\in \mathcal{X}}\| x\| \leq D$  for some  $D < \infty$ . Then, we have
$$
\begin{array}{l} \sup  _ {\theta \in \mathbb {R} ^ {d}} \frac {d _ {\psi} (\delta_ {\theta} P , \delta_ {\theta} \hat {P})}{\hat {V} (\theta)} \\ \leq \frac {2 b}{n} \max  \left\{\psi \cdot \left(4 + 8 \eta^ {2} K _ {1} ^ {2}\right), \right. \\ \end{array}
$$
$$
\left. 1 + \psi \cdot \left(1 + \eta^ {2} \sigma^ {2} + \left(4 + 8 \eta^ {2} K _ {1} ^ {2}\right) \| \theta_ {*} - \hat {\theta} _ {*} \| ^ {2} + 4 \eta^ {2} \sup  _ {x \in \mathcal {X}} \| \nabla f (\theta_ {*}, x) \| ^ {2}\right) \right\}, \tag {E.25}
$$
where  $\hat{\theta}_{*}$  is the minimizer of  $\frac{1}{n}\sum_{i = 1}^{n}\nabla f(\theta ,\hat{x}_i)$  and  $\theta_{*}$  is the minimizer of  $\frac{1}{n}\sum_{i = 1}^{n}\nabla f(\theta ,x_i)$  and  $d_{\psi}$  is the weighted total variation distance defined in Section B.1.
$$
\theta_ {1} = \theta - \frac {\eta}{b} \sum_ {i \in \Omega_ {1}} \nabla f \left(\theta , x _ {i}\right) + \eta \xi_ {1}, \tag {E.26}
$$
$$
\hat {\theta} _ {1} = \theta - \frac {\eta}{b} \sum_ {i \in \Omega_ {1}} \nabla f \left(\theta , \hat {x} _ {i}\right) + \eta \xi_ {1}, \tag {E.27}
$$
$$
\mathbb {E} _ {\xi_ {1}} \left[ V \left(\theta_ {1}\right) \right] = 1 + \mathbb {E} _ {\xi_ {1}} \left[ \left\| \theta_ {1} - \theta_ {*} \right\| ^ {2} \right] = 1 + \eta^ {2} \sigma^ {2} + \left\| \theta - \theta_ {*} - \frac {\eta}{b} \sum_ {i \in \Omega_ {1}} \nabla f (\theta , x _ {i}) \right\| ^ {2}, \tag {E.28}
$$
where  $\mathbb{E}_{\xi_1}$  denotes expectations w.r.t.  $\xi_{1}$  only, and we can further compute that
$$
\begin{array}{l} \left\| \theta - \theta_ {*} - \frac {\eta}{b} \sum_ {i \in \Omega_ {1}} \nabla f (\theta , x _ {i}) \right\| ^ {2} \\ \leq 2 \| \theta - \theta_ {*} \| ^ {2} + \frac {2 \eta^ {2}}{b ^ {2}} \left\| \sum_ {i \in \Omega_ {1}} \nabla f (\theta , x _ {i}) \right\| ^ {2} \\ \leq 2 \| \theta - \theta_ {*} \| ^ {2} + \frac {2 \eta^ {2}}{b ^ {2}} \left(\sum_ {i \in \Omega_ {1}} \| \nabla f (\theta , x _ {i}) \|\right) ^ {2} \\ \leq 2 \| \theta - \theta_ {*} \| ^ {2} + \frac {2 \eta^ {2}}{b ^ {2}} \left(\sum_ {i \in \Omega_ {1}} \| \nabla f (\theta , x _ {i}) - \nabla f (\theta_ {*}, x _ {i}) \| + \| \nabla f (\theta_ {*}, x _ {i}) \|\right) ^ {2} \\ \leq 2 \| \theta - \theta_ {*} \| ^ {2} + \frac {2 \eta^ {2}}{b ^ {2}} \left(b K _ {1} \| \theta - \theta_ {*} \| + b \sup  _ {x \in \mathcal {X}} \| \nabla f (\theta_ {*}, x) \|\right) ^ {2} \\ \leq 2 \| \theta - \theta_ {*} \| ^ {2} + 4 \eta^ {2} K _ {1} ^ {2} \| \theta - \theta_ {*} \| ^ {2} + 4 \eta^ {2} \sup  _ {x \in \mathcal {X}} \| \nabla f (\theta_ {*}, x) \| ^ {2}. \tag {E.29} \\ \end{array}
$$
$$
\mathbb {E} _ {\xi_ {1}} \left[ V \left(\theta_ {1}\right) \right] \leq 1 + \eta^ {2} \sigma^ {2} + 2 \| \theta - \theta_ {*} \| ^ {2} + 4 \eta^ {2} K _ {1} ^ {2} \| \theta - \theta_ {*} \| ^ {2} + 4 \eta^ {2} \sup  _ {x \in \mathcal {X}} \| \nabla f \left(\theta_ {*}, x\right) \| ^ {2}. \tag {E.30}
$$
$$
\mathbb {E} _ {\xi_ {1}} \left[ V \left(\hat {\theta} _ {1}\right) \right] \leq 1 + \eta^ {2} \sigma^ {2} + 2 \| \theta - \theta_ {*} \| ^ {2} + 4 \eta^ {2} K _ {1} ^ {2} \| \theta - \theta_ {*} \| ^ {2} + 4 \eta^ {2} \sup  _ {x \in \mathcal {X}} \| \nabla f (\theta_ {*}, x) \| ^ {2}. \tag {E.31}
$$
Since  $X_{n}$  and  $\hat{X}_{n}$  differ by at most one element and  $\sup_{x\in \mathcal{X}}\| x\| \leq D$  for some  $D < \infty$ , we have  $x_{i} = \hat{x}_{i}$  for any  $i\in \Omega_1$  with probability  $\frac{n - b}{n}$  and  $x_{i}\neq \hat{x}_{i}$  for exactly one  $i\in \Omega_1$  with probability  $\frac{b}{n}$ . Therefore, we have
$$
\left. d _ {\psi} \left(\delta_ {\theta} P, \delta_ {\theta} \hat {P}\right) \leq \frac {2 b}{n} \left(1 + \psi \left(1 + \eta^ {2} \sigma^ {2} + \left(2 + 4 \eta^ {2} K _ {1} ^ {2}\right) \| \theta - \theta_ {*} \| ^ {2} + 4 \eta^ {2} \sup  _ {x \in \mathcal {X}} \| \nabla f \left(\theta_ {*}, x\right) \| ^ {2}\right)\right). \right. \tag {E.32}
$$
$$
\begin{array}{l} \sup  _ {\theta \in \mathbb {R} ^ {d}} \frac {d _ {\psi} (\delta_ {\theta} P , \delta_ {\theta} \hat {P})}{\hat {V} (\theta)} \\ \leq \frac {2 b}{n} \sup _ {\theta \in \mathbb {R} ^ {d}} \frac {1 + \psi \left(1 + \eta^ {2} \sigma^ {2} + (2 + 4 \eta^ {2} K _ {1} ^ {2}) \| \theta - \theta_ {*} \| ^ {2} + 4 \eta^ {2} \sup _ {x \in \mathcal {X}} \| \nabla f (\theta_ {* ,} x) \| ^ {2}\right)}{1 + \| \theta - \hat {\theta} _ {*} \| ^ {2}} \\ \leq \frac {2 b}{n} \sup  _ {\theta \in \mathbb {R} ^ {d}} \left\{\frac {1 + \psi \left(1 + \eta^ {2} \sigma^ {2} + (4 + 8 \eta^ {2} K _ {1} ^ {2}) \| \theta - \hat {\theta} _ {*} \| ^ {2}\right)}{1 + \| \theta - \hat {\theta} _ {*} \| ^ {2}} \right. \\ + \left. \frac {\psi \left((4 + 8 \eta^ {2} K _ {1} ^ {2}) \| \theta_ {*} - \hat {\theta} _ {*} \| ^ {2} + 4 \eta^ {2} \sup _ {x \in \mathcal {X}} \| \nabla f (\theta_ {*} , x) \| ^ {2}\right)}{1 + \| \theta - \hat {\theta} _ {*} \| ^ {2}} \right\} \\ \leq \frac {2 b}{n} \max  \left\{\psi (4 + 8 \eta^ {2} K _ {1} ^ {2}), \right. \\ \left. 1 + \psi \left(1 + \eta^ {2} \sigma^ {2} + \left(4 + 8 \eta^ {2} K _ {1} ^ {2}\right) \| \theta_ {*} - \hat {\theta} _ {*} \| ^ {2} + 4 \eta^ {2} \sup  _ {x \in \mathcal {X}} \| \nabla f (\theta_ {*}, x) \| ^ {2}\right) \right\}. \tag {E.33} \\ \end{array}
$$