$$
\begin{array}{l} \left(\mathbb {E} _ {\mathcal {A}} \left\| \sum_ {t = 0} ^ {T - 1} \left(\frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}) - \nabla_ {\boldsymbol {x}} F _ {\mathcal {S}} (\boldsymbol {x} ^ {t}, \boldsymbol {y} ^ {t})\right) \right\|\right) ^ {2} \\ \leq \mathbb {E} _ {\mathcal {A}} \left\| \sum_ {t = 0} ^ {T - 1} \left(\frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}\right) - \nabla_ {\boldsymbol {x}} F _ {\mathcal {S}} \left(\boldsymbol {x} ^ {t}, \boldsymbol {y} ^ {t}\right)\right) \right\| ^ {2} \\ = \sum_ {t = 0} ^ {T - 1} \mathbb {E} _ {\mathcal {A}} \left\| \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}\right) - \frac {1}{m} \sum_ {i = 1} ^ {m} \frac {1}{n} \sum_ {l = 1} ^ {n} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} ^ {t}, \boldsymbol {y} ^ {t}; \xi_ {i, l}\right) \right\| ^ {2} \\ + \sum_ {t \neq t ^ {\prime}} \mathbb {E} _ {\mathcal {A}} \langle \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}) - \frac {1}{m} \sum_ {i = 1} ^ {m} \frac {1}{n} \sum_ {l = 1} ^ {n} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} ^ {t}, \boldsymbol {y} ^ {t}; \xi_ {i, l}), \\ \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} _ {i} ^ {t ^ {\prime}}, \boldsymbol {y} _ {i} ^ {t ^ {\prime}}; \xi_ {i, j _ {t ^ {\prime}} (i)}\right) - \frac {1}{m} \sum_ {i = 1} ^ {m} \frac {1}{n} \sum_ {l = 1} ^ {n} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} ^ {t ^ {\prime}}, \boldsymbol {y} ^ {t ^ {\prime}}; \xi_ {i, l}\right) \rangle \\ \leq 4 G ^ {2} T + \sum_ {t \neq t ^ {\prime}} \mathbb {E} _ {\mathcal {A}} \left\langle \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}\right) - \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} ^ {t}, \boldsymbol {y} ^ {t}; \xi_ {i, j _ {t} (i)}\right) \right. \\ + \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} ^ {t}, \boldsymbol {y} ^ {t}; \xi_ {i, j _ {t} (i)}) - \frac {1}{m} \sum_ {i = 1} ^ {m} \frac {1}{n} \sum_ {l = 1} ^ {n} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} ^ {t}, \boldsymbol {y} ^ {t}; \xi_ {i, l}), \\ \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} _ {i} ^ {t ^ {\prime}}, \boldsymbol {y} _ {i} ^ {t ^ {\prime}}; \xi_ {i, j _ {t ^ {\prime}} (i)}\right) - \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} ^ {t ^ {\prime}}, \boldsymbol {y} ^ {t ^ {\prime}}; \xi_ {i, j _ {t ^ {\prime}} (i)}\right) \\ + \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} ^ {t ^ {\prime}}, \boldsymbol {y} ^ {t ^ {\prime}}; \xi_ {i, j _ {t ^ {\prime}} (i)}\right) - \frac {1}{m} \sum_ {i = 1} ^ {m} \frac {1}{n} \sum_ {l = 1} ^ {n} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} ^ {t ^ {\prime}}, \boldsymbol {y} ^ {t ^ {\prime}}; \xi_ {i, l}\right) \rangle \\ \stackrel {(a)} {=} 4 G ^ {2} T + \sum_ {t \neq t ^ {\prime}} \mathbb {E} _ {\mathcal {A}} \langle \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}) - \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} (\boldsymbol {x} ^ {t}, \boldsymbol {y} ^ {t}; \xi_ {i, j _ {t} (i)}), \\ \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} _ {i} ^ {t ^ {\prime}}, \boldsymbol {y} _ {i} ^ {t ^ {\prime}}; \xi_ {i, j _ {t ^ {\prime}} (i)}\right) - \frac {1}{m} \sum_ {i = 1} ^ {m} \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} ^ {t ^ {\prime}}, \boldsymbol {y} ^ {t ^ {\prime}}; \xi_ {i, j _ {t ^ {\prime}} (i)}\right) \rangle \\ \leq 4 G ^ {2} T + \sum_ {t \neq t ^ {\prime}} \mathbb {E} _ {\mathcal {A}} \left(\left\| \frac {1}{m} \sum_ {i = 1} ^ {m} \left(\nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} _ {i} ^ {t}, \boldsymbol {y} _ {i} ^ {t}; \xi_ {i, j _ {t} (i)}\right) - \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} ^ {t}, \boldsymbol {y} ^ {t}; \xi_ {i, j _ {t} (i)}\right)\right) \right\|\right). \\ \left. \left\| \frac {1}{m} \sum_ {i = 1} ^ {m} \left(\nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} _ {i} ^ {t ^ {\prime}}, \boldsymbol {y} _ {i} ^ {t ^ {\prime}}; \xi_ {i, j _ {t ^ {\prime}} (i)}\right) - \nabla_ {\boldsymbol {x}} f _ {i} \left(\boldsymbol {x} ^ {t ^ {\prime}}, \boldsymbol {y} ^ {t ^ {\prime}}; \xi_ {i, j _ {t ^ {\prime}} (i)}\right)\right) \right\|\right) \\ \stackrel {(b)} {\leq} 4 G ^ {2} T + \sum_ {t \neq t ^ {\prime}} \frac {L ^ {2}}{m} \mathbb {E} _ {\mathcal {A}} \left[ \sum_ {i = 1} ^ {m} \left(\| \boldsymbol {x} ^ {t} - \boldsymbol {x} _ {i} ^ {t} \| ^ {2} + \| \boldsymbol {y} ^ {t} - \boldsymbol {y} _ {i} ^ {t} \| ^ {2}\right) \cdot \sum_ {i = 1} ^ {m} \left(\| \boldsymbol {x} ^ {t ^ {\prime}} - \boldsymbol {x} _ {i} ^ {t ^ {\prime}} \| ^ {2} + \| \boldsymbol {y} ^ {t ^ {\prime}} - \boldsymbol {y} _ {i} ^ {t ^ {\prime}} \| ^ {2}\right) \right] ^ {1 / 2} \\ \leq 4 G ^ {2} T + \sum_ {t \neq t ^ {\prime}} 4 G ^ {2} L ^ {2} \left(\sum_ {s = 0} ^ {t - 1} \eta_ {s} ^ {m a x} \lambda^ {t - 1 - s}\right) \left(\sum_ {s = 0} ^ {t ^ {\prime} - 1} \eta_ {s} ^ {m a x} \lambda^ {t - 1 - s}\right) \\ \end{array}
$$