SlowGuess/NeurIPS_2023_OCR huggingface dataset:
where  $\hat{\theta}_{*}$  is the minimizer of  $\frac{1}{n}\sum_{i = 1}^{n}\nabla f(\theta ,\hat{x}_i)$  and  $\theta_{*}$  is the minimizer of  $\frac{1}{n}\sum_{i = 1}^{n}\nabla f(\theta ,x_i)$ .
$$
\hat {\theta} _ {k} = \hat {\theta} _ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}). \tag {G.3}
$$
$$
\hat {\theta} _ {k} = \hat {\theta} _ {k - 1} - \frac {\eta}{n} \sum_ {i = 1} ^ {n} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) + \eta \left(\frac {1}{n} \sum_ {i = 1} ^ {n} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) - \frac {1}{b} \sum_ {i \in \Omega_ {k}} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i})\right). \tag {G.4}
$$
$$
\mathbb {E} \left[ \frac {1}{b} \sum_ {i \in \Omega_ {k}} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) | \hat {\theta} _ {k - 1} \right] = \frac {1}{n} \sum_ {i = 1} ^ {n} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}). \tag {G.5}
$$
$$
\begin{array}{l} \mathbb {E} \left\| \hat {\theta} _ {k} - \hat {\theta} _ {*} \right\| ^ {2} = \mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} + \frac {\eta}{n} \sum_ {i = 1} ^ {n} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) \right\| ^ {2} \\ + \eta^ {2} \mathbb {E} \left\| \frac {1}{n} \sum_ {i = 1} ^ {n} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) - \frac {1}{b} \sum_ {i \in \Omega_ {k}} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) \right\| ^ {2}. \tag {G.6} \\ \end{array}
$$
$$
\begin{array}{l} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} + \frac {\eta}{n} \sum_ {i = 1} ^ {n} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) \right\| ^ {2} \\ = \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} + \frac {\eta}{n} \sum_ {i = 1} ^ {n} \left(\nabla f \left(\hat {\theta} _ {k - 1}, \hat {x} _ {i}\right) - \nabla f \left(\hat {\theta} _ {*}, \hat {x} _ {i}\right)\right) \right\| ^ {2} \\ \leq \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {2} - 2 \eta \mu \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {p} + \frac {\eta^ {2}}{n ^ {2}} \left\| \sum_ {i = 1} ^ {n} \left(\nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) - \nabla f (\hat {\theta} _ {*}, \hat {x} _ {i})\right) \right\| ^ {2} \\ \leq \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {2} - 2 \eta \mu \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {p} + \eta^ {2} K _ {1} ^ {2} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {p}. \tag {G.7} \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} \left\| \frac {1}{n} \sum_ {i = 1} ^ {n} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) - \frac {1}{b} \sum_ {i \in \Omega_ {k}} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}) \right\| ^ {2} \\ = \mathbb {E} \left\| \frac {1}{b} \sum_ {i \in \Omega_ {k}} \left(\frac {1}{n} \sum_ {j = 1} ^ {n} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {j}) - \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i})\right) \right\| ^ {2} \\ = \mathbb {E} \left(\frac {1}{b} \sum_ {i \in \Omega_ {k}} K _ {2} \frac {1}{n} \sum_ {j = 1} ^ {n} \| \hat {x} _ {i} - \hat {x} _ {j} \| (2 \| \hat {\theta} _ {k - 1} \| ^ {p - 1} + 1)\right) ^ {2} \\ \leq 4 D ^ {2} K _ {2} ^ {2} \mathbb {E} \left[ (2 \| \hat {\theta} _ {k - 1} \| ^ {p - 1} + 1) ^ {2} \right] \\ \leq 8 D ^ {2} K _ {2} ^ {2} \left(4 \mathbb {E} \left[ \| \hat {\theta} _ {k - 1} \| ^ {2 (p - 1)} \right] + 1\right) \\ \leq 8 D ^ {2} K _ {2} ^ {2} \left(4 \mathbb {E} \left[ \| \hat {\theta} _ {k - 1} \| ^ {p} \right] + 5\right) \\ \leq 8 D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} \mathbb {E} \| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \| ^ {p} + 2 ^ {p + 1} \| \hat {\theta} _ {*} \| ^ {p} + 5\right). \tag {G.8} \\ \end{array}
$$
Hence, by applying (G.7) and (G.8) to (G.6), we conclude that
$$
\begin{array}{l} \mathbb {E} \left\| \hat {\theta} _ {k} - \hat {\theta} _ {*} \right\| ^ {2} \leq \mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {2} - 2 \eta \mu \mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {p} + \eta^ {2} K _ {1} ^ {2} \mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {p} \\ + 8 \eta^ {2} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} \mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {2} + 2 ^ {p + 1} \| \hat {\theta} _ {*} \| ^ {2} + 5\right) \\ \leq \mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {2} - \eta \mu \mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {p} + 8 \eta^ {2} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} \| \hat {\theta} _ {*} \| ^ {p} + 5\right), \tag {G.9} \\ \end{array}
$$
provided that  $\eta \leq \frac{\mu}{K_1^2 + 2^{p + 4}D^2K_2^2}$ . This implies that
$$
\mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {p} \leq \frac {\mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {2} - \mathbb {E} \left\| \hat {\theta} _ {k} - \hat {\theta} _ {*} \right\| ^ {2}}{\eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} \| \hat {\theta} _ {*} \| ^ {p} + 5\right), \tag {G.10}
$$
$$
\begin{array}{l} \frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \left\| \hat {\theta} _ {i - 1} - \hat {\theta} _ {*} \right\| ^ {p} \leq \frac {\left\| \hat {\theta} _ {0} - \hat {\theta} _ {*} \right\| ^ {2} - \mathbb {E} \left\| \hat {\theta} _ {k} - \hat {\theta} _ {*} \right\| ^ {2}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} (2 ^ {p + 1} \| \hat {\theta} _ {*} \| ^ {p} + 5) \\ \leq \frac {\left\| \theta - \hat {\theta} _ {*} \right\| ^ {2}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} \| \hat {\theta} _ {*} \| ^ {p} + 5\right). \tag {G.11} \\ \end{array}
$$
$$
\frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \| \theta_ {i - 1} - \theta_ {*} \| ^ {p} \leq \frac {\| \theta - \theta_ {*} \| ^ {2}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} (2 ^ {p + 1} \| \theta_ {*} \| ^ {p} + 5). \tag {G.12}
$$
Lemma G.2. Under Assumption 4.1, we have
$$
\| \theta_{*}\| \leq \frac{1}{\mu^{\frac{1}{p - 1}}}\sup_{x\in \mathcal{X}}\| \nabla f(0,x)\|^{ \frac{1}{p - 1}},
$$
$$
\| \hat{\theta}_{*}\| \leq \frac{1}{\mu^{\frac{1}{p - 1}}}\sup_{x\in \mathcal{X}}\| \nabla f(0,x)\|^{ \frac{1}{p - 1}}.
$$
Proof. Under Assumption 4.1, we have
$$
\begin{array}{l} \left\langle \nabla \hat {F} (0, X _ {n}) - \nabla \hat {F} (\theta_ {*}, X _ {n}), 0 - \theta_ {*} \right\rangle = - \frac {1}{n} \sum_ {i = 1} ^ {n} \langle \nabla f (0, x _ {i}), \theta_ {*} \rangle \\ = \frac {1}{n} \sum_ {i = 1} ^ {n} \left\langle \nabla f \left(0, x _ {i}\right) - \nabla f \left(\theta_ {*}, x _ {i}\right), 0 - \theta_ {*} \right\rangle \geq \mu \| \theta_ {*} \| ^ {p}, \tag {G.13} \\ \end{array}
$$
where  $p\in (1,2)$ , which implies that
$$
\mu \| \theta_ {*} \| ^ {p} \leq \sup  _ {x \in \mathcal {X}} \| \nabla f (0, x) \| \cdot \| \theta_ {*} \|, \tag {G.14}
$$
$$
\| \theta_ {*} \| \leq \frac {1}{\mu^ {\frac {1}{p - 1}}} \sup  _ {x \in \mathcal {X}} \| \nabla f (0, x) \| ^ {\frac {1}{p - 1}}.
$$
$$
\| \hat {\theta} _ {*} \| \leq \frac {1}{\mu^ {\frac {1}{p - 1}}} \sup  _ {x \in \mathcal {X}} \| \nabla f (0, x) \| ^ {\frac {1}{p - 1}}.
$$
Now, we are able to state the main result for the Wasserstein algorithmic stability.
Theorem G.1. Let  $\theta_0 = \hat{\theta}_0 = \theta$ . Suppose Assumption 4.1 and Assumption 4.2 hold and  $\eta \leq \frac{\mu}{K_1^2 + 2^{p + 4}D^2K_2^2}$  and  $\sup_{x\in \mathcal{X}}\| x\| \leq D$  for some  $D < \infty$  and  $\sup_{x\in \mathcal{X}}\| \nabla f(0,x)\| \leq E$  for some  $E < \infty$ . Let  $\nu_{k}$  and  $\hat{\nu}_{k}$  denote the distributions of  $\theta_{k}$  and  $\hat{\theta}_k$  respectively. Then, we have
$$
\begin{array}{l} \frac {1}{k} \sum_ {i = 1} ^ {k} \mathcal {W} _ {p} ^ {p} (\nu_ {i - 1}, \hat {\nu} _ {i - 1}) \\ \leq \frac {4 D ^ {2} K _ {2} ^ {2} \eta}{b n \mu} \cdot 2 ^ {p + 2} \cdot \left(\frac {2 \theta^ {2} + 2 (E / \mu) ^ {\frac {2}{p - 1}}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} (E / \mu) ^ {\frac {p}{p - 1}} + 5\right)\right) \\ + \frac {4 D ^ {2} K _ {2} ^ {2} \eta}{b n \mu} \cdot 2 ^ {p + 2} \left(2 ^ {p + 2} (E / \mu) ^ {\frac {p}{p - 1}} + 1 0\right) \\ + \frac {4 D K _ {2}}{n \mu} (1 + K _ {1} \eta) \cdot 3 \cdot 2 ^ {p - 1} \left(\frac {2 \theta^ {2} + 2 (E / \mu) ^ {\frac {2}{p - 1}}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} (E / \mu) ^ {\frac {p}{p - 1}} + 5\right)\right) \\ + \frac {4 D K _ {2}}{n \mu} (1 + K _ {1} \eta) \cdot 7 \cdot 2 ^ {p - 1} \left(\frac {2 \theta^ {2} + 2 (E / \mu) ^ {\frac {2}{p - 1}}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} (E / \mu) ^ {\frac {p}{p - 1}} + 5\right)\right) \\ + \frac {4 D K _ {2}}{n \mu} \left(1 + K _ {1} \eta\right) \left(1 0 \cdot 2 ^ {p - 1} (E / \mu) ^ {\frac {p}{p - 1}} + 5\right). \tag {G.15} \\ \end{array}
$$
$$
\theta_ {k} = \theta_ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \nabla f \left(\theta_ {k - 1}, x _ {i}\right), \tag {G.16}
$$
$$
\hat {\theta} _ {k} = \hat {\theta} _ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \nabla f (\hat {\theta} _ {k - 1}, \hat {x} _ {i}). \tag {G.17}
$$
$$
\theta_ {k} - \hat {\theta} _ {k} = \theta_ {k - 1} - \hat {\theta} _ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \left(\nabla f \left(\theta_ {k - 1}, x _ {i}\right) - \nabla f \left(\hat {\theta} _ {k - 1}, x _ {i}\right)\right) + \frac {\eta}{b} \mathcal {E} _ {k}, \tag {G.18}
$$
$$
\mathcal {E} _ {k} := \sum_ {i \in \Omega_ {k}} \left(\nabla f \left(\hat {\theta} _ {k - 1}, \hat {x} _ {i}\right) - \nabla f \left(\hat {\theta} _ {k - 1}, x _ {i}\right)\right). \tag {G.19}
$$
$$
\begin{array}{l} \left\| \theta_ {k} - \hat {\theta} _ {k} \right\| ^ {2} = \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \left(\nabla f (\theta_ {k - 1}, x _ {i}) - \nabla f (\hat {\theta} _ {k - 1}, x _ {i})\right) \right\| ^ {2} + \frac {\eta^ {2}}{b ^ {2}} \| \mathcal {E} _ {k} \| ^ {2} \\ + 2 \left\langle \theta_ {k - 1} - \hat {\theta} _ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \left(\nabla f \left(\theta_ {k - 1}, x _ {i}\right) - \nabla f \left(\hat {\theta} _ {k - 1}, x _ {i}\right)\right), \frac {\eta}{b} \mathcal {E} _ {k} \right\rangle . \tag {G.20} \\ \end{array}
$$
By Assumption 4.2 and Assumption 4.1, we have
$$
\begin{array}{l} \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \left(\nabla f \left(\theta_ {k - 1}, x _ {i}\right) - \nabla f \left(\hat {\theta} _ {k - 1}, x _ {i}\right)\right) \right\| ^ {2} \\ \leq \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {2} - 2 \eta \mu \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {p} + \frac {\eta^ {2}}{b ^ {2}} \left(b K _ {1} \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {\frac {p}{2}}\right) ^ {2} \\ = \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {2} - 2 \eta \mu \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {p} + \eta^ {2} K _ {1} ^ {2} \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {p} \\ \leq \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {2} - \eta \mu \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {p}, \tag {G.21} \\ \end{array}
$$
provided that  $\eta \leq \frac{\mu}{K_1^2}$ .
$$
\begin{array}{l} \mathbb {E} \left\| \mathcal {E} _ {k} \right\| ^ {2} \leq \frac {b}{n} \mathbb {E} \left[ \left(K _ {2} 2 D (2 \| \hat {\theta} _ {k - 1} \| ^ {p - 1} + 1)\right) ^ {2} \right] \\ \leq \frac {4 D ^ {2} K _ {2} ^ {2} b}{n} \left(8 \mathbb {E} \left[ \| \hat {\theta} _ {k - 1} \| ^ {2 (p - 1)} \right] + 2\right) \\ \leq \frac {4 D ^ {2} K _ {2} ^ {2} b}{n} \left(8 \mathbb {E} \left[ \| \hat {\theta} _ {k - 1} \| ^ {p} \right] + 1 0\right) \\ \leq \frac {4 D ^ {2} K _ {2} ^ {2} b}{n} \left(2 ^ {p + 2} \mathbb {E} \| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \| ^ {p} + 2 ^ {p + 2} \| \hat {\theta} _ {*} \| ^ {p} + 1 0\right), \tag {G.22} \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} \left\langle \theta_ {k - 1} - \hat {\theta} _ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \left(\nabla f \left(\theta_ {k - 1}, x _ {i}\right) - \nabla f \left(\hat {\theta} _ {k - 1}, x _ {i}\right)\right), \frac {\eta}{b} \mathcal {E} _ {k} \right\rangle \\ \leq \frac {\eta}{b} \frac {b}{n} \mathbb {E} \left[ \left(\left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| + K _ {1} \eta \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {\frac {p}{2}}\right) K _ {2} 2 D \left(2 \| \hat {\theta} _ {k - 1} \| ^ {p - 1} + 1\right) \right] \\ \leq \frac {2 D K _ {2} \eta}{n} \mathbb {E} \left[ \left((1 + K _ {1} \eta) \| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \| + K _ {1} \eta\right) \left(2 \| \hat {\theta} _ {k - 1} \| ^ {p - 1} + 1\right) \right] \\ \leq \frac {2 D K _ {2} \eta}{n} (1 + K _ {1} \eta) \mathbb {E} \left[ \left(\| \theta_ {k - 1} \| + \| \hat {\theta} _ {k - 1} \| + 1\right) \left(2 \| \hat {\theta} _ {k - 1} \| ^ {p - 1} + 1\right) \right]. \tag {G.23} \\ \end{array}
$$
Notice that for any  $x, y \geq 0$  and  $p \in (1,2)$ , we have  $xy^{p - 1} \leq x^p + y^p$ ,  $y \leq y^p + 1$ ,  $x \leq x^p + 1$  and  $y^{p - 1} \leq y^p + 1$ , which implies that
$$
(x + y + 1) \left(2 y ^ {p - 1} + 1\right) = 2 x y ^ {p - 1} + 2 y ^ {p} + 2 y ^ {p - 1} + x + y + 1 \leq 3 x ^ {p} + 7 y ^ {p} + 5. \tag {G.24}
$$
Therefore, by applying (G.24) to (G.23), we have
$$
\begin{array}{l} \mathbb {E} \left\langle \theta_ {k - 1} - \hat {\theta} _ {k - 1} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \left(\nabla f \left(\theta_ {k - 1}, x _ {i}\right) - \nabla f \left(\hat {\theta} _ {k - 1}, x _ {i}\right)\right), \frac {\eta}{b} \mathcal {E} _ {k} \right\rangle \\ \leq \frac {2 D K _ {2} \eta}{n} (1 + K _ {1} \eta) \left(3 \mathbb {E} \| \theta_ {k - 1} \| ^ {p} + 7 \mathbb {E} \| \hat {\theta} _ {k - 1} \| ^ {p} + 5\right) \\ \leq \frac {2 D K _ {2} \eta}{n} (1 + K _ {1} \eta) \left(3 \cdot 2 ^ {p - 1} \mathbb {E} \| \theta_ {k - 1} - \theta_ {*} \| ^ {p} + 3 \cdot 2 ^ {p - 1} \| \theta_ {*} \| ^ {p} \right. \\ \left. + 7 \cdot 2 ^ {p - 1} \mathbb {E} \| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \| ^ {p} + 7 \cdot 2 ^ {p - 1} \| \hat {\theta} _ {*} \| ^ {p} + 5\right). \tag {G.25} \\ \end{array}
$$
Hence, by applying (G.21), (G.22), (G.25) into (G.20), we conclude that
$$
\begin{array}{l} \mathbb {E} \left\| \theta_ {k} - \hat {\theta} _ {k} \right\| ^ {2} \leq \mathbb {E} \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {2} - \eta \mu \mathbb {E} \left\| \theta_ {k - 1} - \hat {\theta} _ {k - 1} \right\| ^ {p} \\ + \frac {4 D ^ {2} K _ {2} ^ {2} \eta^ {2}}{b n} \left(2 ^ {p + 2} \mathbb {E} \left\| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \right\| ^ {p} + 2 ^ {p + 2} \| \hat {\theta} _ {*} \| ^ {p} + 1 0\right) \\ + \frac {4 D K _ {2} \eta}{n} (1 + K _ {1} \eta) \left(3 \cdot 2 ^ {p - 1} \mathbb {E} \| \theta_ {k - 1} - \theta_ {*} \| ^ {p} + 3 \cdot 2 ^ {p - 1} \| \theta_ {*} \| ^ {p} \right. \\ + 7 \cdot 2 ^ {p - 1} \mathbb {E} \| \hat {\theta} _ {k - 1} - \hat {\theta} _ {*} \| ^ {p} + 7 \cdot 2 ^ {p - 1} \| \hat {\theta} _ {*} \| ^ {p} + 5), \tag {G.26} \\ \end{array}
$$
provided that  $\eta \leq \frac{\mu}{K_1^2}$ . This, together with  $\theta_0 = \hat{\theta}_0 = \theta$ , implies that
$$
\begin{array}{l} \frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \left\| \theta_ {i - 1} - \hat {\theta} _ {i - 1} \right\| ^ {p} \\ \leq \frac {4 D ^ {2} K _ {2} ^ {2} \eta}{b n \mu} \left(2 ^ {p + 2} \frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \left\| \hat {\theta} _ {i - 1} - \hat {\theta} _ {*} \right\| ^ {p} + 2 ^ {p + 2} \| \hat {\theta} _ {*} \| ^ {p} + 1 0\right) \\ + \frac {4 D K _ {2}}{n \mu} (1 + K _ {1} \eta) \left(3 \cdot 2 ^ {p - 1} \frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \| \theta_ {i - 1} - \theta_ {*} \| ^ {p} + 3 \cdot 2 ^ {p - 1} \| \theta_ {*} \| ^ {p} \right. \\ + 7 \cdot 2 ^ {p - 1} \frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \| \hat {\theta} _ {i - 1} - \hat {\theta} _ {*} \| ^ {p} + 7 \cdot 2 ^ {p - 1} \| \hat {\theta} _ {*} \| ^ {p} + 5). \tag {G.27} \\ \end{array}
$$
In Lemma G.1, we showed that under the assumption  $\eta \leq \frac{\mu}{K_1^2 + 2^{p + 4}D^2K_2^2}$ , we have
$$
\frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \| \theta_ {i - 1} - \theta_ {*} \| ^ {p} \leq \frac {\| \theta - \theta_ {*} \| ^ {2}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} (2 ^ {p + 1} \| \theta_ {*} \| ^ {p} + 5), \tag {G.28}
$$
$$
\frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \left\| \hat {\theta} _ {i - 1} - \hat {\theta} _ {*} \right\| ^ {p} \leq \frac {\left\| \theta - \hat {\theta} _ {*} \right\| ^ {2}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} \| \hat {\theta} _ {*} \| ^ {p} + 5\right). \tag {G.29}
$$
Hence, by plugging (G.28) and (G.29) into (G.27), we conclude that
$$
\begin{array}{l} \frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \left\| \theta_ {i - 1} - \hat {\theta} _ {i - 1} \right\| ^ {p} \\ \leq \frac {4 D ^ {2} K _ {2} ^ {2} \eta}{b n \mu} \left(2 ^ {p + 2} \left(\frac {\left\| \theta - \hat {\theta} _ {*} \right\| ^ {2}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} \| \hat {\theta} _ {*} \| ^ {p} + 5\right)\right) + 2 ^ {p + 2} \| \hat {\theta} _ {*} \| ^ {p} + 1 0\right) \\ + \frac {4 D K _ {2}}{n \mu} (1 + K _ {1} \eta) \cdot 3 \cdot 2 ^ {p - 1} \left(\frac {\| \theta - \theta_ {*} \| ^ {2}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} \| \theta_ {*} \| ^ {p} + 5\right)\right) \\ + \frac {4 D K _ {2}}{n \mu} (1 + K _ {1} \eta) \cdot 7 \cdot 2 ^ {p - 1} \left(\frac {\left\| \theta - \hat {\theta} _ {*} \right\| ^ {2}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} \| \hat {\theta} _ {*} \| ^ {p} + 5\right)\right) \\ + \frac {4 D K _ {2}}{n \mu} \left(1 + K _ {1} \eta\right) \left(3 \cdot 2 ^ {p - 1} \| \theta_ {*} \| ^ {p} + 7 \cdot 2 ^ {p - 1} \| \hat {\theta} _ {*} \| ^ {p} + 5\right). \tag {G.30} \\ \end{array}
$$
Moreover,  $\| \theta - \hat{\theta}_* \|^2 \leq 2 \| \theta \|^2 + 2 \| \hat{\theta}_* \|^2$ ,  $\| \theta - \theta_* \|^2 \leq 2 \| \theta \|^2 + 2 \| \theta_* \|^2$  and by applying Lemma G.2, we obtain
$$
\begin{array}{l} \frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \left\| \theta_ {i - 1} - \hat {\theta} _ {i - 1} \right\| ^ {p} \\ \leq \frac {4 D ^ {2} K _ {2} ^ {2} \eta}{b n \mu} \left(2 ^ {p + 2} \left(\frac {2 \theta^ {2} + 2 (E / \mu) ^ {\frac {2}{p - 1}}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} (E / \mu) ^ {\frac {p}{p - 1}} + 5\right)\right) \right. \\ \left. + 2 ^ {p + 2} (E / \mu) ^ {\frac {p}{p - 1}} + 1 0\right) \\ + \frac {4 D K _ {2}}{n \mu} (1 + K _ {1} \eta) \cdot 3 \cdot 2 ^ {p - 1} \left(\frac {2 \theta^ {2} + 2 (E / \mu) ^ {\frac {2}{p - 1}}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} (E / \mu) ^ {\frac {p}{p - 1}} + 5\right)\right) \\ + \frac {4 D K _ {2}}{n \mu} (1 + K _ {1} \eta) \cdot 7 \cdot 2 ^ {p - 1} \left(\frac {2 \theta^ {2} + 2 (E / \mu) ^ {\frac {2}{p - 1}}}{k \eta \mu} + \frac {8 \eta}{\mu} D ^ {2} K _ {2} ^ {2} \left(2 ^ {p + 1} (E / \mu) ^ {\frac {p}{p - 1}} + 5\right)\right) \\ + \frac {4 D K _ {2}}{n \mu} \left(1 + K _ {1} \eta\right) \left(1 0 \cdot 2 ^ {p - 1} (E / \mu) ^ {\frac {p}{p - 1}} + 5\right). \tag {G.31} \\ \end{array}
$$
Finally, by the definition of  $p$ -Wasserstein distance, we have
$$
\frac {1}{k} \sum_ {i = 1} ^ {k} \mathcal {W} _ {p} ^ {p} \left(\nu_ {i - 1}, \hat {\nu} _ {i - 1}\right) \leq \frac {1}{k} \sum_ {i = 1} ^ {k} \mathbb {E} \left\| \theta_ {i - 1} - \theta_ {*} \right\| ^ {p}. \tag {G.32}
$$
G.1 Proof of Theorem 4.2
Proof. First, let us show that the sequences  $\theta_{k}$  and  $\hat{\theta}_k$  are ergodic.
First, let us show that the limit  $\theta_{\infty}$ , if exists, is unique. Consider two sequences  $\theta_k^{(1)}$  and  $\theta_k^{(2)}$  starting at  $\theta_0^{(1)}$  and  $\theta_0^{(2)}$  respectively with two limits  $\theta_{\infty}^{(1)}$  and  $\theta_{\infty}^{(2)}$ . For any  $k \in \mathbb{N}$
$$
\theta_ {k} ^ {(1)} = \theta_ {k - 1} ^ {(1)} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \nabla f \left(\theta_ {k - 1} ^ {(1)}, x _ {i}\right), \tag {G.33}
$$
$$
\theta_ {k} ^ {(2)} = \theta_ {k - 1} ^ {(2)} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \nabla f \left(\theta_ {k - 1} ^ {(2)}, x _ {i}\right). \tag {G.34}
$$
$$
\begin{array}{l} \left\| \theta_ {k} ^ {(2)} - \theta_ {k} ^ {(1)} \right\| ^ {2} = \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} - \frac {\eta}{b} \sum_ {i \in \Omega_ {k}} \left(\nabla f \left(\theta_ {k - 1} ^ {(2)}, x _ {i}\right) - \nabla f \left(\theta_ {k - 1} ^ {(1)}, x _ {i}\right)\right) \right\| ^ {2} \\ \leq \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {2} - 2 \eta \mu \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {p} + \frac {\eta^ {2}}{b ^ {2}} \left(b K _ {1} \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {\frac {p}{2}}\right) ^ {2} \\ = \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {2} - 2 \eta \mu \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {p} + \eta^ {2} K _ {1} ^ {2} \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {p} \\ \leq \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {2} - \eta \mu \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {p}, \tag {G.35} \\ \end{array}
$$
provided that  $\eta \leq \frac{\mu}{K_1^2}$ . This implies that
$$
\left\| \theta_ {k} ^ {(2)} - \theta_ {k} ^ {(1)} \right\| ^ {2} \leq \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {2} - \eta \mu \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {p}, \tag {G.36}
$$
provided that  $\eta \leq \frac{\mu}{K_1^2}$ . Thus,  $\left\| \theta_{k}^{(2)} - \theta_{k}^{(1)} \right\|^2 \leq \left\| \theta_{k-1}^{(2)} - \theta_{k-1}^{(1)} \right\|^2$  for any  $k \in \mathbb{N}$ . Suppose  $\left\| \theta_{j-1}^{(2)} - \theta_{j-1}^{(1)} \right\|^2 \geq 1$  for every  $j = 1, 2, \ldots, k$ , then we have
$$
\left\| \theta_ {k} ^ {(2)} - \theta_ {k} ^ {(1)} \right\| ^ {2} \leq \left\| \theta_ {0} ^ {(2)} - \theta_ {0} ^ {(1)} \right\| ^ {2} - k \eta \mu , \tag {G.37}
$$
$$
\left\| \theta_ {k} ^ {(2)} - \theta_ {k} ^ {(1)} \right\| ^ {2} \leq 1, \quad \text {f o r a n y} k \geq k _ {0} := \frac {\left\| \theta_ {0} ^ {(2)} - \theta_ {0} ^ {(1)} \right\| ^ {2} - 1}{\eta \mu}. \tag {G.38}
$$
Since  $p\in (1,2)$
$$
\left\| \theta_ {k} ^ {(2)} - \theta_ {k} ^ {(1)} \right\| ^ {2} \leq (1 - \eta \mu) \left\| \theta_ {k - 1} ^ {(2)} - \theta_ {k - 1} ^ {(1)} \right\| ^ {2}, \tag {G.39}
$$
for any  $k \geq k_0 + 1$ , which implies that  $\theta_k^{(2)} - \theta_k^{(1)} \to 0$  as  $k \to \infty$  so that  $\theta_{\infty}^{(2)} = \theta_{\infty}^{(1)}$ .
Next, let us show that for any sequence  $\theta_{k}$ , it converges to a limit. It follows from (G.39) that
$$
\left\| \theta_ {k} ^ {(2)} - \theta_ {k} ^ {(1)} \right\| ^ {2} \leq (1 - \eta \mu) ^ {k - \lceil k _ {0} \rceil} \left\| \theta_ {\lceil k _ {0} \rceil} ^ {(2)} - \theta_ {\lceil k _ {0} \rceil} ^ {(1)} \right\| ^ {2}, \tag {G.40}
$$
for any  $k \geq k_0 + 1$ . Let  $\theta_0^{(1)}$  be a fixed initial value in  $\mathbb{R}^d$ , and let  $\theta_0^{(2)} = \theta_1^{(1)}$  which is random yet takes only finitely many values given  $\theta_0^{(1)}$  so that  $k_0$  is bounded. Therefore, it follows from (G.40) that
$$
\mathcal {W} _ {2} ^ {2} \left(\nu_ {k + 1}, \nu_ {k}\right) \leq (1 - \eta \mu) ^ {k} \mathbb {E} \left[ \left(1 - \eta \mu\right) ^ {- \lceil k _ {0} \rceil} \left\| \theta_ {\lceil k _ {0} \rceil} ^ {(2)} - \theta_ {\lceil k _ {0} \rceil} ^ {(1)} \right\| ^ {2} \right], \tag {G.41}
$$
where  $\nu_{k}$  denotes the distribution of  $\theta_{k}$ , which implies that
$$
\sum_ {k = 1} ^ {\infty} \mathcal {W} _ {2} ^ {2} \left(\nu_ {k + 1}, \nu_ {k}\right) <   \infty . \tag {G.42}
$$
Thus,  $(\nu_{k})$  is a Cauchy sequence in  $\mathcal{P}_2(\mathbb{R}^d)$  equipped with metric  $\mathcal{W}_2$  and hence there exists some  $\nu_{\infty}$  such that  $\mathcal{W}_2(\nu_k,\nu_\infty)\to 0$  as  $k\to \infty$