SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \epsilon_ {g e n} ^ {w} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) \\ = (\sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} \mathbb {E} [ F (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}) ] - \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} \mathbb {E} [ F (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) ]) - (\sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} \mathbb {E} [ F _ {\mathcal {S}} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S})), \boldsymbol {y} ^ {\prime}) ] - \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} \mathbb {E} [ F _ {\mathcal {S}} (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) ]) \\ = \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} \mathbb {E} [ F (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}) ] - \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} \mathbb {E} [ F _ {\mathcal {S}} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S})), \boldsymbol {y} ^ {\prime}) ] + \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} \mathbb {E} [ F _ {\mathcal {S}} (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) ] - \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} \mathbb {E} [ F (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) ] \\ \leq \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} \left(\mathbb {E} _ {\mathcal {A}, \mathcal {S}} \left[ F \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}\right) - F _ {\mathcal {S}} \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}\right) \right]\right) + \sup  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} \left(\mathbb {E} _ {\mathcal {A}, \mathcal {S}} \left[ F _ {\mathcal {S}} \left(\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})\right) - F \left(\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})\right) \right]\right) \\ \end{array}
$$
where the expectation can be detailed into  $\mathbb{E}_{A,S}$  in this situation.
$$
\begin{array}{l} \mathbb {E} _ {\mathcal {A}, \mathcal {S}} \left[ F \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}\right) - F _ {\mathcal {S}} \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}\right) \right] \\ = \frac {1}{n ^ {m}} \sum_ {l _ {1} = 1} ^ {n} \dots \sum_ {l _ {m} = 1} ^ {n} \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} \left[ F \left(\mathcal {A} _ {\boldsymbol {x}} \left(\mathcal {S} ^ {(l)}\right), \boldsymbol {y} ^ {\prime}\right) \right] - \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} \left[ F _ {\mathcal {S}} \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}\right) \right] \\ = \frac {1}{n ^ {m}} \sum_ {l _ {1} = 1} ^ {n} \dots \sum_ {l _ {m} = 1} ^ {n} \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} [ \frac {1}{m} \sum_ {i = 1} ^ {m} f _ {i} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {\boldsymbol {l}}), \boldsymbol {y} ^ {\prime}; \xi_ {i, l _ {i}}) - \frac {1}{m} \sum_ {i = 1} ^ {m} f _ {i} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}; \xi_ {i, l _ {i}}) ] \\ \leq \frac {1}{n ^ {m}} \sum_ {l _ {1} = 1} ^ {n} \dots \sum_ {l _ {m} = 1} ^ {n} \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} [ \frac {1}{m} \sum_ {i = 1} ^ {m} G \| \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {l}) - \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}) \| ] \\ = G \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} \| \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {l}) - \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}) \| \\ \end{array}
$$
where the first equation is due to the symmetric distribution between  $S_{i}$  and  $S_{i}^{\prime}$  and there are overall  $n^{m}$  permutations of  $l$ . And the second equation is due to the independence of sample  $\xi_{i,l_i}$  from the training process upon dataset  $S^{l}$  and Lemma 4.
And we can analyze the second term in a similar way:
$$
\mathbb {E} _ {\mathcal {A}, \mathcal {S}} \left[ F _ {\mathcal {S}} \left(\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})\right) - F \left(\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})\right) \right] \leq G \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} \left\| \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S}) - \mathcal {A} _ {\boldsymbol {y}} \left(\mathcal {S} ^ {l}\right) \right\|
$$
Therefore we can get the weak primal-dual generalization gap that:
$$
\epsilon_ {g e n} ^ {w} \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})\right) \leq \sqrt {2} G \sup  _ {\mathcal {S}, \mathcal {S} ^ {\prime}} \mathbb {E} _ {\mathcal {A}} \left\| \begin{array}{c} \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}) - \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {l}) \\ \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S}) - \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S} ^ {l}) \end{array} \right\| \leq \sqrt {2} G \epsilon
$$
For case b. the strong primal-dual generalization gap  $\epsilon_{gen}^{s}(\mathcal{A}_{\boldsymbol{x}}(\mathcal{S}),\mathcal{A}_{\boldsymbol{y}}(\mathcal{S})) = \Delta^{s}(\mathcal{A}_{\boldsymbol{x}}(\mathcal{S}),\mathcal{A}_{\boldsymbol{y}}(\mathcal{S})) - \Delta_{\mathcal{S}}^{s}(\mathcal{A}_{\boldsymbol{x}}(\mathcal{S}),\mathcal{A}_{\boldsymbol{y}}(\mathcal{S}))$
Observing the structure that:
$$
\begin{array}{l} \epsilon_ {g e n} ^ {s} \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})\right) \\ = \mathbb {E} [ \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} F (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}) - \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} F (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) ] - \mathbb {E} [ \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} F _ {\mathcal {S}} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}) - \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} F _ {\mathcal {S}} (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) ] \\ = \mathbb {E} _ {\mathcal {A}, S} [ \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} F (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}) - \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} F _ {\mathcal {S}} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}) ] + \mathbb {E} _ {\mathcal {A}, S} [ \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} F _ {\mathcal {S}} (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) - \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} F (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) ] \tag {6} \\ \end{array}
$$
where the expectation can be detailed into  $\mathbb{E}_{S,A}$  in this situation.
$$
\begin{array}{l} \mathbb {E} _ {\mathcal {A}, \mathcal {S}} [ \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} F (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}) ] \stackrel {(i)} {=} \frac {1}{n ^ {m}} \sum_ {l _ {1} = 1} ^ {n} \sum_ {l _ {2} = 1} ^ {n} \dots \sum_ {l _ {m} = 1} ^ {n} \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} [ \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} F (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {(l)}), \boldsymbol {y} ^ {\prime}) ] \\ \stackrel {(i i)} {=} \frac {1}{n ^ {m}} \sum_ {l _ {1} = 1} ^ {n} \sum_ {l _ {2} = 1} ^ {n} \dots \sum_ {l _ {m} = 1} ^ {n} \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} [ F (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {(l)}), \boldsymbol {y} _ {\mathcal {S} ^ {(l)}} ^ {*}) ] \\ \stackrel {(i i i)} {=} \frac {1}{n ^ {m}} \sum_ {l _ {1} = 1} ^ {n} \sum_ {l _ {2} = 1} ^ {n} \dots \sum_ {l _ {m} = 1} ^ {n} \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} [ \frac {1}{m} \sum_ {i = 1} ^ {m} f _ {i} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {(l)}), \boldsymbol {y} _ {\mathcal {S} ^ {(l)}} ^ {*}; \xi_ {i, l _ {i}}) ] \\ \end{array}
$$
Since  $S_{i}$  and  $S_{i}^{\prime}$  are drawn from the same distribution, and there are  $n^{m}$  permutations of  $l = \{l_1,\dots,l_m\}$  so we can get equality (i). In equality (ii), we denote  $\pmb{y}_{S^{(l)}}^{*}$  as  $\arg \max_{y\in \mathcal{Y}}F(\mathcal{A}_x(S),\pmb {y})$ .
While for equality  $(iii)$ ,  $\xi_{i,l_i}$  is independent from the training process under dataset  $S^l$  for each local agent respectively.
Using the property of Lipschitz continuous (see Assumption 1), we can further get:
$$
\begin{array}{l} f _ {i} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {(l)}), \boldsymbol {y} _ {\mathcal {S} ^ {(l)}} ^ {*}; \xi_ {i, l _ {i}}) - f _ {i} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} _ {\mathcal {S}} ^ {*}; \xi_ {i, l _ {i}}) \leq G \left\| \left( \begin{array}{c} \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {(l)}) - \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}) \\ \boldsymbol {y} _ {\mathcal {S} ^ {(l)}} ^ {*} - \boldsymbol {y} _ {\mathcal {S}} ^ {*} \end{array} \right) \right\| \\ \stackrel {(a)} {\leq} G \sqrt {1 + \frac {L ^ {2}}{\mu_ {\boldsymbol {y}} ^ {2}}} \| \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {l}) - \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}) \| \\ \end{array}
$$
where inequality  $(a)$  is a conclusion of Lemma 4.3 in [22] that  $\| \pmb{y}_{\mathcal{S}^{(l)}}^* - \pmb{y}_S^* \| \leq \frac{L}{\mu_y} \| \mathcal{A}_x(\mathcal{S}^{(l)}) - \mathcal{A}_x(\mathcal{S})\|$ .
Combining above two inequalities, we can get:
$$
\begin{array}{l} \mathbb{E}_{\mathcal{A},\mathcal{S}}[\sup_{\boldsymbol {y}^{\prime}\in \mathcal{Y}}F(\mathcal{A}_{\boldsymbol{x}}(\mathcal{S}),\boldsymbol{y}^{\prime})] \\ \leq \frac {1}{n ^ {m}} \sum_ {l _ {1} = 1} ^ {n} \sum_ {l _ {2} = 1} ^ {n} \dots \sum_ {l _ {m} = 1} ^ {n} \mathbb {E} _ {\mathcal {A}, \mathcal {S}, \mathcal {S} ^ {\prime}} \left[ \frac {1}{m} \sum_ {i = 1} ^ {m} \left(f _ {i} \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} _ {\mathcal {S}} ^ {*}; \xi_ {i, l _ {i}}\right) + G \sqrt {1 + \frac {L ^ {2}}{\mu_ {\boldsymbol {y}} ^ {2}}} \| \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {l}) - \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}) \|\right) \right] \\ = \mathbb {E} _ {\mathcal {A}, \mathcal {S}} \left[ F _ {\mathcal {S}} \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} _ {\mathcal {S}} ^ {*}\right) \right] + G \sqrt {1 + \frac {L ^ {2}}{\mu_ {\boldsymbol {y}} ^ {2}}} \mathbb {E} _ {\mathcal {A}} \left[ \sup  _ {\mathcal {S}, \mathcal {S} ^ {\prime}} \| \mathcal {A} _ {\boldsymbol {x}} \left(\mathcal {S} ^ {l}\right) - \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}) \| \right] \\ \leq \mathbb {E} _ {\mathcal {A}, \mathcal {S}} \left[ \sup  _ {\boldsymbol {y} ^ {\prime} \in \mathcal {Y}} F _ {\mathcal {S}} \left(\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \boldsymbol {y} ^ {\prime}\right) \right] + G \sqrt {1 + \frac {L ^ {2}}{\mu_ {\boldsymbol {y}} ^ {2}}} \mathbb {E} _ {\mathcal {A}} \left[ \sup  _ {\mathcal {S}, \mathcal {S} ^ {\prime}} \| \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {l}) - \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}) \| \right] \\ \end{array}
$$
where we get the last but one inequality from Lemma 4.
We can do a similar operation on the second counterpart since  $f_{i}$  is also  $\mu_{\pmb{x}}$ -strongly convex on  $\pmb{x}$ .
$$
\mathbb {E} _ {\mathcal {A}, \mathcal {S}} \left[ \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} F _ {\mathcal {S}} (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) - \inf  _ {\boldsymbol {x} ^ {\prime} \in \mathcal {X}} F (\boldsymbol {x} ^ {\prime}, \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) \right] \leq G \sqrt {1 + \frac {L ^ {2}}{\mu_ {\boldsymbol {x}} ^ {2}}} \mathbb {E} _ {\mathcal {A}} \left[ \sup  _ {\mathcal {S}, \mathcal {S} ^ {\prime}} \| \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S} ^ {l}) - \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S}) \| \right]
$$
So the overall strong primal-dual generalization gap satisfies:
$$
\begin{array}{l} \epsilon_ {g e n} ^ {s} (\mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}), \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S})) \\ \leq G \sqrt {1 + \frac {L ^ {2}}{\mu^ {2}}} \sqrt {2} \mathbb {E} _ {\mathcal {A}} \left[ \sup  _ {\mathcal {S}, \mathcal {S} ^ {\prime}} \left\| \begin{array}{c} \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S}) - \mathcal {A} _ {\boldsymbol {x}} (\mathcal {S} ^ {l}) \\ \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S}) - \mathcal {A} _ {\boldsymbol {y}} (\mathcal {S} ^ {l}) \end{array} \right. \right\| \Bigg ] \\ \leq G \sqrt {2 + \frac {2 L ^ {2}}{\mu^ {2}}} \epsilon \\ \end{array}
$$
where  $\mu = \min \{\mu_{x},\mu_{y}\}$
E Proof in the Strongly-Convex-Strongly-Concave Case
In this section, we will prove the stability and generalization gap of our D-SGDA in the case of  $\mu_{\pmb{x}}$ -strongly convex and  $\mu_{\pmb{y}}$ -strongly concave.
E.1 Proof of Stability
Proof of Theorem 2. We use  $(\pmb{x}^t,\pmb{y}^t)$  and  $(\dot{\pmb{x}}^{t},\dot{\pmb{y}}^{t})$  to represent the output in the  $t$ -th iteration when applying D-SGDA on any arbitrary neighbouring dataset  $\mathcal{S}$  and  $\mathcal{S}'$  respectively. Since the D-SGDA algorithm is symmetric [4] with respect to the dataset  $\mathcal{S} = \{\mathcal{S}_1,\dots,\mathcal{S}_m\}$ , we can assume the different samples appear in the last location in each  $\mathcal{S}_i$  without loss of generalization, i.e.,  $\mathcal{S}' = \{\mathcal{S}_1',\dots,\mathcal{S}_{m - 1}',\mathcal{S}_m'\}$  where  $\mathcal{S}_i' = \{\xi_{i,1},\xi_{i,2},\dots,\xi_{i,n - 1},\xi_{i,n}'\}$  differs from  $\bar{\mathcal{S}}_i = \{\xi_{i,1},\dots,\xi_{i,n}\}$  in the  $n$ -th data.
First concentrating on the iteration, we have:
$$
\begin{array}{l} \left( \begin{array}{c} \boldsymbol {x} ^ {t + 1} \\ \boldsymbol {y} ^ {t + 1} \end{array} \right) = \left([ \boldsymbol {X}, \boldsymbol {Y} ] ^ {t + 1}\right) ^ {T} \frac {\mathbb {1} _ {m}}{m} \\ = P _ {(\mathcal {X}, \mathcal {Y}) ^ {m}} \left[ \left(\boldsymbol {W} [ \boldsymbol {X}, \boldsymbol {Y} ] ^ {t} - \left( \begin{array}{c} \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} \boldsymbol {f} (\boldsymbol {X} ^ {t}, \boldsymbol {Y} ^ {t}; \boldsymbol {\xi} ^ {t}) \\ - \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} \boldsymbol {f} (\boldsymbol {X} ^ {t}, \boldsymbol {Y} ^ {t}; \boldsymbol {\xi} ^ {t}) \end{array} \right)\right) ^ {T} \right] \frac {\mathbb {1} _ {m}}{m} \\ = P _ {(\mathcal {X}, \mathcal {Y})} \left[ \left(\boldsymbol {W} [ \boldsymbol {X}, \boldsymbol {Y} ] ^ {t} - \left( \begin{array}{c} \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} \boldsymbol {f} (\boldsymbol {X} ^ {t}, \boldsymbol {Y} ^ {t}; \boldsymbol {\xi} ^ {t}) \\ - \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} \boldsymbol {f} (\boldsymbol {X} ^ {t}, \boldsymbol {Y} ^ {t}; \boldsymbol {\xi} ^ {t}) \end{array} \right)\right) ^ {T} \frac {\mathbb {1} _ {m}}{m} \right] \\ = P _ {(\mathcal {X}, \mathcal {Y})} \left[ \left([ \boldsymbol {X}, \boldsymbol {Y} ] ^ {t}\right) ^ {T} \boldsymbol {W} ^ {T} \frac {\mathbb {1} _ {m}}{m} - \left( \begin{array}{c} \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} \boldsymbol {f} (\boldsymbol {X} ^ {t}, \boldsymbol {Y} ^ {t}; \boldsymbol {\xi} ^ {t}) \\ - \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} \boldsymbol {f} (\boldsymbol {X} ^ {t}, \boldsymbol {Y} ^ {t}; \boldsymbol {\xi} ^ {t}) \end{array} \right) ^ {T} \frac {\mathbb {1} _ {m}}{m} \right] \\ = P _ {(\mathcal {X}, \mathcal {Y})} \left[ \left( \begin{array}{c} \boldsymbol {x} ^ {t} \\ \boldsymbol {y} ^ {t} \end{array} \right) - \left( \begin{array}{c} \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} \boldsymbol {f} (\boldsymbol {X} ^ {t}, \boldsymbol {Y} ^ {t}; \boldsymbol {\xi} ^ {t}) \\ - \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} \boldsymbol {f} (\boldsymbol {X} ^ {t}, \boldsymbol {Y} ^ {t}; \boldsymbol {\xi} ^ {t}) \end{array} \right) ^ {T} \frac {\mathbb {1} _ {m}}{m} \right] \\ = P _ {(\mathcal {X}, \mathcal {Y})} \left[ \left( \begin{array}{c} \boldsymbol {x} ^ {t} \\ \boldsymbol {y} ^ {t} \end{array} \right) - \frac {1}{m} \sum_ {i = 1} ^ {m} \left( \begin{array}{c} \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}) \\ - \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}) \end{array} \right) \right] \\ \end{array}
$$
where in the first equality we use the definition that  $\pmb{x}^t = \frac{1}{m}\sum_{i=1}^{m}\pmb{x}_i^t$ ,  $\pmb{y}^t = \frac{1}{m}\sum_{i=1}^{m}\pmb{y}_i^t$ , and the last equality is due to the fact that  $\pmb{W}\frac{\mathbb{1}_m}{m} = \frac{\mathbb{1}_m}{m}$ . We use the notation  $P_{(\mathcal{X},\mathcal{Y})}\left[\begin{pmatrix}\pmb{x}\\ \pmb{y}\end{pmatrix}\right] \in \mathbb{R}^{d_x + d_y}$  as an abbreviation of  $\left( \begin{array}{c}P_{\mathcal{X}}[\pmb{x}]\\ P_{\mathcal{Y}}[\pmb{y}] \end{array} \right) \in \mathbb{R}^{d_x + d_y}$ , and  $P_{(\mathcal{X},\mathcal{Y})^m}[[\pmb{X},\pmb{Y}]^T]$  means every column should obey  $P_{(\mathcal{X},\mathcal{Y})}\left[\begin{pmatrix}\pmb{x}\\ \pmb{y}\end{pmatrix}\right]$ .
In the  $t$ -th iteration, there is a probability of  $C_m^{m_0}(1 - \frac{1}{n})^{m - m_0}(\frac{1}{n})^{m_0}$  that there are  $m - m_0$  agents not selecting the last different samples while the rest  $m_0$  agents selecting exactly the last different samples. Without loss of generalization, we assume the first  $m - m_0$  not selecting and the rest  $m_0$  agents selecting the different samples respectively. So we can get:
$$
\begin{array}{l} \left( \begin{array}{c} \boldsymbol {x} ^ {t + 1} - \dot {\boldsymbol {x}} ^ {t + 1} \\ \boldsymbol {y} ^ {t + 1} - \dot {\boldsymbol {y}} ^ {t + 1} \end{array} \right) \\ = P _ {\mathcal {X}, \mathcal {Y}} \left[ \left( \begin{array}{c} \boldsymbol {x} ^ {t} - \dot {\boldsymbol {x}} ^ {t} \\ \boldsymbol {y} ^ {t} - \dot {\boldsymbol {y}} ^ {t} \end{array} \right) - \frac {1}{m} \sum_ {i = 1} ^ {m} \left( \begin{array}{c} \eta_ {\boldsymbol {x}, t} (\nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}) - \nabla_ {\boldsymbol {x}} f _ {i} (\dot {\boldsymbol {x}} _ {i} ^ {t}, \dot {\boldsymbol {y}} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)})) \\ - \eta_ {\boldsymbol {y}, t} (\nabla_ {\boldsymbol {y}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}) - \nabla_ {\boldsymbol {y}} f _ {i} (\dot {\boldsymbol {x}} _ {i} ^ {t}, \dot {\boldsymbol {y}} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)})) \end{array} \right) \right] \\ \stackrel {{(i)}} {{=}} P _ {\mathcal {X}, \mathcal {Y}} \left[ \right.\left(\begin{array}{c}\boldsymbol {x} ^ {t} - \dot {\boldsymbol {x}} ^ {t}\\\boldsymbol {y} ^ {t} - \dot {\boldsymbol {y}} ^ {t}\end{array}\right) - \frac {1}{m} \left[ \sum_ {i = 1} ^ {m - m _ {0}} \left(\begin{array}{c}\eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}) - \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} f _ {i} (\dot {\boldsymbol {x}} _ {i} ^ {t}, \dot {\boldsymbol {y}} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)})\\- \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}) + \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} f _ {i} (\dot {\boldsymbol {x}} _ {i} ^ {t}, \dot {\boldsymbol {y}} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)})\end{array}\right)\right] \\ \left. \left. + \sum_ {i = m - m _ {0} + 1} ^ {m} \left( \begin{array}{c} \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, n}) - \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} f _ {i} (\dot {\boldsymbol {x}} _ {i} ^ {t}, \dot {\boldsymbol {y}} _ {i} ^ {t}; \xi_ {i, n} ^ {\prime}) \\ - \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, n}) + \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} f _ {i} (\dot {\boldsymbol {x}} _ {i} ^ {t}, \dot {\boldsymbol {y}} _ {i} ^ {t}; \xi_ {i, n} ^ {\prime}) \end{array} \right) \right] \right] \\ \end{array}
$$
where equality  $(i)$  is due to the setting that agents from 1 to  $m - m_0$  select  $\xi_{i,j_i(i)}$  which is not the last different one while the rest agents from  $m - m_0 + 1$  to  $m$  select the last different samples, i.e.,  $\xi_{i,n}$  and  $\xi_{i,n}^{\prime}$  respectively.
Then we can decompose the term inside the projection into several parts as following:
According to Lemma 1 and the preproperty of Lipschitz continuity (see Assumption 1), we can bound the term  $I_{1}$  as:
$$
\| I _ {1} \| \leq \frac {m - m _ {0}}{m} (1 - \eta_ {t} ^ {m i n} \frac {L \mu}{L + \mu}) \left\| \left( \begin{array}{c} \boldsymbol {x} ^ {t} - \dot {\boldsymbol {x}} ^ {t} \\ \boldsymbol {y} ^ {t} - \dot {\boldsymbol {y}} ^ {t} \end{array} \right) \right\|
$$
where  $\eta_t^{min} \triangleq \min \{\eta_{\pmb{x},t}, \eta_{\pmb{y},t}\}$ ,  $\eta_t^{max} \triangleq \max \{\eta_{\pmb{x},t}, \eta_{\pmb{y},t}\}$ , and  $\mu \triangleq \min \{\mu_{\pmb{x}}, \mu_{\pmb{y}}\}$ . For the term  $I_2$ , it is bounded by:
$$
\begin{array}{l} \| I_{2}\| \leq \frac{1}{m}\sum_{i = m - m_{0} + 1}^{m}\bigg\|\bigg( \begin{array}{c}\boldsymbol{x}^{t} - \eta_{\boldsymbol {x},t}\nabla_{\boldsymbol{x}}f_{i}(\boldsymbol{x}^{t},\boldsymbol{y}^{t};\xi_{i,n}) - \big(\dot{\boldsymbol{x}}^{t} - \eta_{\boldsymbol {x},t}\nabla_{\boldsymbol{x}}f_{i}(\dot{\boldsymbol{x}}^{t},\dot{\boldsymbol{y}}^{t};\xi^{\prime}_{i,n})\big)\\ \boldsymbol{y}^{t} + \eta_{\boldsymbol {y},t}\nabla_{\boldsymbol{y}}f_{i}(\boldsymbol{x}^{t},\boldsymbol{y}^{t};\xi_{i,n}) - \big(\dot{\boldsymbol{y}}^{t} + \eta_{\boldsymbol {y},t}\nabla_{\boldsymbol{y}}f_{i}(\dot{\boldsymbol{x}}^{t},\dot{\boldsymbol{y}}^{t};\xi^{\prime}_{i,n})\big)\bigg)\bigg\| \\ \leq \frac{1}{m}\sum_{i = m - m_{0} + 1}^{m}\left[ \right.\left\|\left(\begin{array}{c}\boldsymbol{x}^{t} - \eta_{\boldsymbol {x},t}\nabla_{\boldsymbol{x}}f_{i}(\boldsymbol{x}^{t},\boldsymbol{y}^{t};\xi_{i,n}) - (\dot{\boldsymbol{x}}^{t} - \eta_{\boldsymbol {x},t}\nabla_{\boldsymbol{x}}f_{i}(\dot{\boldsymbol{x}}^{t},\dot{\boldsymbol{y}}^{t};\xi_{i,n}))\\\boldsymbol{y}^{t} + \eta_{\boldsymbol {y},t}\nabla_{\boldsymbol{y}}f_{i}(\boldsymbol{x}^{t},\boldsymbol{y}^{t};\xi_{i,n}) - (\dot{\boldsymbol{y}}^{t} + \eta_{\boldsymbol {y},t}\nabla_{\boldsymbol{y}}f_{i}(\dot{\boldsymbol{x}}^{t},\dot{\boldsymbol{y}}^{t};\xi_{i,n}))\end{array}\right)\right\| \\ \left. + \left\| \left( \begin{array}{c} \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} f _ {i} (\dot {\boldsymbol {x}} ^ {t}, \dot {\boldsymbol {y}} ^ {t}; \xi_ {i, n} ^ {\prime}) - \eta_ {\boldsymbol {x}, t} \nabla_ {\boldsymbol {x}} f _ {i} (\dot {\boldsymbol {x}} ^ {t}, \dot {\boldsymbol {y}} ^ {t}; \xi_ {i, n}) \\ \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} f _ {i} (\dot {\boldsymbol {x}} ^ {t}, \dot {\boldsymbol {y}} ^ {t}; \xi_ {i, n}) - \eta_ {\boldsymbol {y}, t} \nabla_ {\boldsymbol {y}} f _ {i} (\dot {\boldsymbol {x}} ^ {t}, \dot {\boldsymbol {y}} ^ {t}; \xi_ {i, n} ^ {\prime}) \end{array} \right) \right\| \right] \\ \leq \frac {m _ {0}}{m} \left[ \left(1 - \eta_ {t} ^ {m i n} \frac {L \mu}{L + \mu}\right) \left\| \left( \begin{array}{c} \boldsymbol {x} ^ {t} - \dot {\boldsymbol {x}} ^ {t} \\ \boldsymbol {y} ^ {t} - \dot {\boldsymbol {y}} ^ {t} \end{array} \right) \right\| + 2 \eta_ {t} ^ {m a x} G \right] \\ \end{array}
$$
For the term  $II_{1}$  and  $II_1^\prime$