SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} f _ {i} \left(\frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} (h _ {j} (\hat {\mathbf {w}} _ {t}) - h _ {i} (\hat {\mathbf {w}} _ {t}), \hat {s} _ {i, t}), \hat {s} _ {t} ^ {\prime}\right) - f _ {i} (u _ {i, t}, s _ {t} ^ {\prime}) \\ \stackrel {(a)} {\geq} \partial_ {s ^ {\prime}} f _ {i} (u _ {i, t}, s _ {t} ^ {\prime}) (\hat {s} _ {t} ^ {\prime} - s _ {t} ^ {\prime}) + \partial_ {u} f _ {i} (u _ {i, t}, s _ {t} ^ {\prime}) (\frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} (h _ {j} (\hat {\mathbf {w}} _ {t}) - h _ {i} (\hat {\mathbf {w}} _ {t}), \hat {s} _ {i, t}) - u _ {i, t}) \\ \stackrel {(b)} {\geq} \partial_ {s ^ {\prime}} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \left(\hat {s} _ {t} ^ {\prime} - s _ {t} ^ {\prime}\right) + \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \left[ \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) - u _ {i, t} \right. \\ + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \left\langle \partial_ {v} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right), \left(h _ {j} \left(\hat {\mathbf {w}} _ {t}\right) - h _ {i} \left(\hat {\mathbf {w}} _ {t}\right)\right) - \left(v _ {j, t} - v _ {i, t}\right)) \right\rangle \\ - \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \frac {\rho_ {g}}{2} \left\| \left(h _ {j} \left(\hat {\mathbf {w}} _ {t}\right) - h _ {i} \left(\hat {\mathbf {w}} _ {t}\right)\right) - \left(v _ {j, t} - v _ {i, t}\right) \right\| ^ {2} \\ \left. + \left\langle \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \partial_ {s _ {i}} g _ {i} \left(v _ {j, t} - v _ {i, t}\right), s _ {i, t}, \hat {s} _ {i, t} - s _ {i, t} \right\rangle - \frac {\rho_ {g}}{2} \| \hat {s} _ {i, t} - s _ {i, t} \| ^ {2} \right] \tag {38} \\ \stackrel {(c)} {\geq} \partial_ {s ^ {\prime}} f _ {i} (u _ {i, t}, s _ {t} ^ {\prime}) (\hat {s} _ {t} ^ {\prime} - s _ {t} ^ {\prime}) + \partial_ {u} f _ {i} (u _ {i, t}, s _ {t} ^ {\prime}) \bigg [ \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} (v _ {j, t} - v _ {i, t}, s _ {i, t}) - u _ {i, t} \bigg ] \\ + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \underbrace {\left\langle \right. \partial_ {u} f _ {i} \left(u _ {i , t} , s _ {t} ^ {\prime}\right) \partial_ {v} g _ {i} \left(v _ {j , t} - v _ {i , t} , s _ {i , t}\right) , \left(h _ {j} \left(\hat {\mathbf {w}} _ {t}\right) - h _ {i} \left(\hat {\mathbf {w}} _ {t}\right)\right) - \left(v _ {j , t} - v _ {i , t}\right)\left. \right)\left. \right\rangle} _ {A _ {1}} \\ + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \left\langle \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \partial_ {s _ {i}} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right), \hat {s} _ {i, t} - s _ {i, t} \right\rangle \\ - \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \frac {\rho_ {g} C _ {f}}{2} \| (h _ {j} (\hat {\mathbf {w}} _ {t}) - h _ {i} (\hat {\mathbf {w}} _ {t})) - (v _ {j, t} - v _ {i, t}) \| ^ {2} - \frac {\rho_ {g} C _ {f}}{2} \| \hat {s} _ {i, t} - s _ {i, t} \| ^ {2} \\ \end{array}
$$
where (a) follows from the convexity of  $f_{i}$ , (b) follows from the monotonicity of  $f_{i}(\cdot, s')$  and weak convexity of  $g_{i}$ , (c) is due to  $0 \leq \partial_{u}f_{i}(u_{i,t}, s_{t}') \leq C_{f}$ .
The  $L_{h}$ -smoothness assumption of  $h_i(\mathbf{w}) - h_j(\mathbf{w})$  for all  $i, \mathbf{w}$  implies
$$
\begin{array}{l} h _ {i} (\hat {\mathbf {w}} _ {t}) - h _ {j} (\hat {\mathbf {w}} _ {t}) \\ \geq h _ {i} \left(\mathbf {w} _ {t}\right) - h _ {j} \left(\mathbf {w} _ {t}\right) + \left\langle \left(\nabla h _ {i} \left(\mathbf {w} _ {t}\right) - \nabla h _ {j} \left(\mathbf {w} _ {t}\right)\right), \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \right\rangle - \frac {L _ {h}}{2} \left\| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \right\| ^ {2} \tag {39} \\ \end{array}
$$
Since  $\partial_{u}f_{i}(u_{i,t},s_{t}^{\prime})\partial_{v}g_{i}(v_{j,t} - v_{i,t},s_{i,t})\geq 0$  , we bound  $A_{1}$  as following
$$
A _ {1} = \left\langle \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \partial_ {v} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right), \left(h _ {j} \left(\hat {\mathbf {w}} _ {t}\right) - h _ {i} \left(\hat {\mathbf {w}} _ {t}\right)\right) - \left(v _ {j, t} - v _ {i, t}\right) \right\rangle
$$
$$
\begin{array}{l} \stackrel {(a)} {\geq} \left\langle \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \partial_ {v} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right), \left(h _ {i} \left(\mathbf {w} _ {t}\right) - h _ {j} \left(\mathbf {w} _ {t}\right)\right) - \left(v _ {j, t} - v _ {i, t})\right) \right\rangle \\ - \left\langle \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \partial_ {v} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right), \frac {L _ {h}}{2} \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} \right\rangle \\ + \left\langle \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \partial_ {v} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) \left(\nabla h _ {i} (\mathbf {w} _ {t}) - \nabla h _ {j} (\mathbf {w} _ {t})\right), \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \right\rangle \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(b)} {\geq} - C _ {f} C _ {g} \left[ \left\| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \right\| + \left\| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \right\| \right] - \frac {C _ {f} C _ {g} L _ {h}}{2} \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} \\ + \left\langle \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \partial_ {\ell} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) \left(\nabla h _ {i} (\mathbf {w} _ {t}) - \nabla h _ {j} (\mathbf {w} _ {t})\right), \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \right\rangle \\ \end{array}
$$
where inequality (a) follows from inequality 39, (b) follows from the Lipschitz continuity and monotone assumptions on  $f_{i}, g_{i}, h_{i}, h_{j}$ . Then plugging the new formulation of  $A_{1}$  back to inequality 38
$$
\begin{array}{l} f _ {i} (\frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} (h _ {j} (\hat {\mathbf {w}} _ {t}) - h _ {i} (\hat {\mathbf {w}} _ {t}), \hat {s} _ {i, t}), \hat {s} _ {t} ^ {\prime}) - f _ {i} (u _ {i, t}, s _ {t} ^ {\prime}) \\ \geq \partial_ {s ^ {\prime}} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \left(\hat {s} _ {t} ^ {\prime} - s _ {t} ^ {\prime}\right) + \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \left[ \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) - u _ {i, t} \right] \\ + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \left[ - C _ {f} C _ {g} \left[ \left\| h _ {i} \left(\mathbf {w} _ {t}\right) - v _ {i, t} \right\| + \left\| h _ {j} \left(\mathbf {w} _ {t}\right) - v _ {j, t} \right\| \right] \right] - \frac {C _ {f} C _ {g} L _ {h}}{2} \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} \\ + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \left\langle \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \partial_ {v} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) \left(\nabla h _ {i} \left(\mathbf {w} _ {t}\right) - \nabla h _ {j} \left(\mathbf {w} _ {t}\right)\right), \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \right\rangle \\ + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \left\langle \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \partial_ {s _ {i}} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right), \hat {s} _ {i, t} - s _ {i, t} \right\rangle \\ - \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \frac {\rho_ {g} C _ {f}}{2} \left\| \left(h _ {j} (\hat {\mathbf {w}} _ {t}) - h _ {i} (\hat {\mathbf {w}} _ {t})\right) - \left(v _ {j, t} - v _ {i, t}\right) \right\| ^ {2} - \frac {\rho_ {g} C _ {f}}{2} \| \hat {s} _ {i, t} - s _ {i, t} \| ^ {2} \\ \end{array}
$$
Taking average over  $i \in S_{+}$  gives
$$
\begin{array}{l} \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} f _ {i} (\frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} (h _ {j} (\hat {\mathbf {w}} _ {t}) - h _ {i} (\hat {\mathbf {w}} _ {t}), \hat {s} _ {i, t}), \hat {s} _ {t} ^ {\prime}) - f _ {i} (u _ {i, t}, s _ {t} ^ {\prime}) \\ \geq \langle \mathbb {E} _ {t} [ G _ {t} ^ {2} ], \hat {s} _ {t} ^ {\prime} - s _ {t} ^ {\prime} \rangle + \langle \mathbb {E} _ {t} [ G _ {t} ], \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \rangle + \langle \mathbb {E} _ {t} [ G _ {t} ^ {1} ], \hat {\mathbf {s}} _ {t} - \mathbf {s} _ {t} \rangle \\ + \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \left[ \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) - u _ {i, t} \right] \\ - C _ {f} C _ {g} \left[ \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| \right] - \frac {C _ {f} C _ {g} L _ {h}}{2} \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} \\ - \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \frac {\rho_ {g} C _ {f}}{2} \| (h _ {j} (\hat {\mathbf {w}} _ {t}) - h _ {i} (\hat {\mathbf {w}} _ {t})) - (v _ {j, t} - v _ {i, t}) \| ^ {2} - \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \frac {\rho_ {g} C _ {f}}{2} \| \hat {s} _ {i, t} - s _ {i, t} \| ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \langle \mathbb {E} _ {t} [ G _ {t} ^ {2} ], \hat {s} _ {t} ^ {\prime} - s _ {t} ^ {\prime} \rangle + \langle \mathbb {E} _ {t} [ G _ {t} ], \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \rangle + \langle \mathbb {E} _ {t} [ G _ {t} ^ {1} ], \hat {\mathbf {s}} _ {t} - \mathbf {s} _ {t} \rangle \\ \leq \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \left[ f _ {i} \left(\frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(h _ {j} (\hat {\mathbf {w}} _ {t}) - h _ {i} (\hat {\mathbf {w}} _ {t}), \hat {s} _ {i, t}\right), \hat {s} _ {t} ^ {\prime}\right) - f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \right. \\ - \partial_ {u} f _ {i} \left(u _ {i, t}, s _ {t} ^ {\prime}\right) \left[ \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) - u _ {i, t} \right] \\ \left. + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \frac {\rho_ {g} C _ {f}}{2} \| \left(h _ {j} (\hat {\mathbf {w}} _ {t}) - h _ {i} (\hat {\mathbf {w}} _ {t})\right) - \left(v _ {j, t} - v _ {i, t}\right) \| ^ {2} + \frac {\rho_ {g} C _ {f}}{2} \| \hat {s} _ {i, t} - s _ {i, t} \| ^ {2} \right] \\ + C _ {f} C _ {g} \left[ \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| \right] + \frac {C _ {f} C _ {g} L _ {h}}{2} \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} \tag {40} \\ \end{array}
$$
Combining inequality 37 and 40 yields
where (a) follows from the Lipschitz continuity of  $f_{i}, g_{i}, h_{i}, h_{j}$  and inequality 40.
Due to the  $\rho_F$ -weak convexity of  $F(\mathbf{w},\mathbf{s}_i,s')$ , we have  $(\bar{\rho} -\rho_F)$ -strong convexity of  $(\mathbf{w},s_i,s')\mapsto F(\mathbf{w},\mathbf{s},s') + \frac{\bar{\rho}}{2}\| (\mathbf{w}_t,\mathbf{s}_t,s'_t) - (\mathbf{w},\mathbf{s},s')\|^2$ . Then it follows
$$
\begin{array}{l} F (\hat {\mathbf {w}} _ {t}, \hat {\mathbf {s}} _ {t}, \hat {s} _ {t} ^ {\prime}) - F _ {i} (\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}) = \left[ F _ {i} (\hat {\mathbf {w}} _ {t}, \hat {\mathbf {s}} _ {t}, \hat {s} _ {t} ^ {\prime}) + \frac {\bar {\rho}}{2} \| (\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}) - (\hat {\mathbf {w}} _ {t}, \hat {\mathbf {s}} _ {t}, \hat {s} _ {t} ^ {\prime}) \| ^ {2} \right] \\ - \left[ F _ {i} \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) + \frac {\bar {\rho}}{2} \left\| \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) - \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) \right\| ^ {2} \right] \tag {42} \\ - \frac {\bar {\rho}}{2} \left\| \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) - \left(\hat {\mathbf {w}} _ {t}, \hat {\mathbf {s}} _ {t}, \hat {s} _ {t} ^ {\prime}\right) \right\| ^ {2} \\ \leq \left(\frac {\rho_ {F}}{2} - \bar {\rho}\right) \| \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) - \left(\hat {\mathbf {w}} _ {t}, \hat {\mathbf {s}} _ {t}, \hat {s} _ {t} ^ {\prime}\right) \| ^ {2} \\ \end{array}
$$
Plugging inequality 42 back into 41, we obtain
$$
\begin{array}{l} \mathbb {E} _ {t} \left[ F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {t + 1}, \mathbf {s} _ {t + 1}, s _ {t + 1} ^ {\prime}\right) \right] \\ \leq F _ {1 / \bar {\rho}} (\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}) + \frac {3 \eta^ {2} \bar {\rho} M ^ {2}}{2} + \bar {\rho} \eta \bigg \{\frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \bigg [ (\frac {\rho_ {F}}{2} - \bar {\rho}) \| (\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}) - (\hat {\mathbf {w}} _ {t}, \hat {\mathbf {s}} _ {t}, \hat {s} _ {t} ^ {\prime}) \| ^ {2} \\ + \frac {2 C _ {f} C _ {g}}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \left[ \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| + \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| \right] \\ + \frac {2 \rho_ {g} C _ {f}}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \left[ \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| ^ {2} + \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| ^ {2} \right] \\ \left. + 2 C _ {f} \left\| \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} (v _ {j, t} - v _ {i, t}, s _ {i, t}) - u _ {i, t} \right\| + \frac {\rho_ {g} C _ {f}}{2} \| \hat {s} _ {i, t} - s _ {i, t} \| ^ {2} \right] + (4 \rho_ {g} C _ {f} C _ {h} + \frac {C _ {f} C _ {g} L _ {h}}{2}) \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} \Biggr \} \\ \leq F _ {1 / \bar {\rho}} (\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}) + \frac {3 \eta^ {2} \bar {\rho} M ^ {2}}{2} + \bar {\rho} \eta \bigg \{\frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \bigg [ - \frac {\bar {\rho}}{2} \| (\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}) - (\hat {\mathbf {w}} _ {t}, \hat {\mathbf {s}} _ {t}, \hat {s} _ {t} ^ {\prime}) \| ^ {2} \\ + \frac {C _ {1}}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \left[ \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| + \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| + \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| ^ {2} + \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| ^ {2} \right] \\ \left. + C _ {1} \left\| \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) - u _ {i, t} \right\| \right] \Bigg \} \\ \stackrel {(b)} {\leq} F _ {1 / \bar {\rho}} (\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}) + \frac {3 \eta^ {2} \bar {\rho} M ^ {2}}{2} - \frac {\eta}{2} \| \nabla F _ {1 / \bar {\rho}} (\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}) \| ^ {2} \\ + \frac {\bar {\rho} \eta C _ {1}}{n _ {+} n _ {-}} \sum_ {X _ {i} \in S _ {+}} \sum_ {X _ {j} \in S _ {-}} \left[ \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| + \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| + \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| ^ {2} + \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| ^ {2} \right] \\ + \frac {\bar {\rho} \eta C _ {1}}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \left\| \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) - u _ {i, t} \right\| \\ \end{array}
$$
where in inequality (a) we use  $\bar{\rho} = \rho_F + \rho_g C_f + 8\rho_g C_f C_h + C_f C_g L_h$  and  $C_1 = \max\{2C_f C_g, 2\rho_g C_f, 2C_f\}$ , and inequality (b) uses Lemma 3.2.
$$
\begin{array}{l} \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \mathbb {E} [ \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| ] \leq (1 - \mu_ {1}) ^ {t} \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {0}) - v _ {i, 0} \| + R _ {1}, \\ \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \mathbb {E} [ \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| ] \leq (1 - \mu_ {2}) ^ {t} \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {0}) - v _ {j, 0} \| + R _ {2}, \\ \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \mathbb {E} [ \| h _ {i} (\mathbf {w} _ {t}) - v _ {i, t} \| ^ {2} ] \leq (1 - \mu_ {1}) ^ {t} \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {0}) - v _ {i, 0} \| ^ {2} + R _ {3}, \\ \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \mathbb {E} [ \| h _ {j} (\mathbf {w} _ {t}) - v _ {j, t} \| ^ {2} ] \leq (1 - \mu_ {2}) ^ {t} \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {0}) - v _ {j, 0} \| ^ {2} + R _ {4}, \\ \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \mathbb {E} \left[ \left\| \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {j, t} - v _ {i, t}, s _ {i, t}\right) - u _ {i, t} \right\| \right] \\ \leq \left(1 - \mu_ {3}\right) ^ {t} \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \left\| \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {i, 0} - v _ {j, 0}, s _ {i, 0}\right) - u _ {i, 0} \right\| + R _ {5}, \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} \left[ F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {t + 1}, \mathbf {s} _ {t + 1}, s _ {t + 1} ^ {\prime}\right) \right] \\ \leq \mathbb {E} \left[ F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) \right] + \frac {3 \eta^ {2} \bar {\rho} M ^ {2}}{2} - \frac {\eta}{2} \mathbb {E} \left[ \| \nabla F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) \| ^ {2} \right] \\ + \bar {\rho} \eta C _ {1} \left[ (1 - \mu_ {1}) ^ {t} \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {0}) - v _ {i, 0} \| + (1 - \mu_ {2}) ^ {t} \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {0}) - v _ {j, 0} \| \right. \\ + \left(1 - \mu_ {1}\right) ^ {t} \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \left\| h _ {i} \left(\mathbf {w} _ {0}\right) - v _ {i, 0} \right\| ^ {2} + \left(1 - \mu_ {2}\right) ^ {t} \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \left\| h _ {j} \left(\mathbf {w} _ {0}\right) - v _ {j, 0} \right\| ^ {2} \\ + (1 - \mu_ {3}) ^ {t} \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \left\| \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} (v _ {i, 0} - v _ {j, 0}, s _ {i, 0}) - u _ {i, 0} \right\| + R _ {1} + R _ {2} + R _ {3} + R _ {4} + R _ {5} \Bigg ] \\ \leq \mathbb {E} \left[ F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) \right] + \frac {3 \eta^ {2} \bar {\rho} M ^ {2}}{2} - \frac {\eta}{2} \mathbb {E} \left[ \| \nabla F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) \| ^ {2} \right] \\ + \bar {\rho} \eta C _ {1} \left[ (1 - \mu_ {m i n}) ^ {t} \left(\frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {0}) - v _ {i, 0} \| + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {0}) - v _ {j, 0} \right. \right. \\ + \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {0}) - v _ {i, 0} \| ^ {2} + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {0}) - v _ {j, 0} \| ^ {2} \\ \left. + \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \left\| \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {i, 0} - v _ {j, 0}, s _ {i, 0}\right) - u _ {i, 0} \right\|\right) + R _ {1} + R _ {2} + R _ {3} + R _ {4} + R _ {5} \\ \end{array}
$$
where  $\mu_{min} = \min \{\mu_1,\mu_2,\mu_3\}$
$$
\begin{array}{l} \mathbb {E} \left[ F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {T}, \mathbf {s} _ {T}, s _ {T} ^ {\prime}\right) \right] \\ \leq F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {0}, \mathbf {s} _ {0}, s _ {0} ^ {\prime}\right) + \frac {3 \eta^ {2} \bar {\rho} M ^ {2} T}{2} - \frac {\eta}{2} \sum_ {t = 0} ^ {T - 1} \mathbb {E} \left[ \| \nabla F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) \| ^ {2} \right] \\ + \bar {\rho} \eta C _ {1} \left[ \sum_ {t = 0} ^ {T - 1} (1 - \mu_ {m i n}) ^ {t} \left(\frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {0}) - v _ {i, 0} \| + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {0}) - v _ {j, 0} \right. \right. \\ + \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {0}) - v _ {i, 0} \| ^ {2} + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {0}) - v _ {j, 0} \| ^ {2} \\ \left. + \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \left\| \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {i, 0} - v _ {j, 0}, s _ {i, 0}\right) - u _ {i, 0} \right\|\right) + T \left(R _ {1} + R _ {2} + R _ {3} + R _ {4} + R _ {5}\right) \\ \leq F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {0}, \mathbf {s} _ {0}, s _ {0} ^ {\prime}\right) + \frac {3 \eta^ {2} \bar {\rho} M ^ {2} T}{2} - \frac {\eta}{2} \sum_ {t = 0} ^ {T - 1} \| \nabla F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) \| ^ {2} \\ + \bar {\rho} \eta C _ {1} \left[ \frac {\Delta_ {0}}{\mu_ {m i n}} + T \left(R _ {1} + R _ {2} + R _ {3} + R _ {4} + R _ {5}\right) \right] \\ \end{array}
$$
$$
\begin{array}{l} \left(\frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {0}) - v _ {i, 0} \| + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {0}) - v _ {j, 0} \right\| \\ + \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \| h _ {i} (\mathbf {w} _ {0}) - v _ {i, 0} \| ^ {2} + \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} \| h _ {j} (\mathbf {w} _ {0}) - v _ {j, 0} \| ^ {2} \\ \left. + \frac {1}{n _ {+}} \sum_ {X _ {i} \in S _ {+}} \left\| \frac {1}{n _ {-}} \sum_ {X _ {j} \in S _ {-}} g _ {i} \left(v _ {i, 0} - v _ {j, 0}, s _ {i, 0}\right) - u _ {i, 0} \right\|\right) \leq \Delta_ {0}. \\ \end{array}
$$
$$
\begin{array}{l} \frac {1}{T} \sum_ {t = 0} ^ {T - 1} \| \nabla F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {t}, \mathbf {s} _ {t}, s _ {t} ^ {\prime}\right) \| ^ {2} \\ \leq \frac {2}{\eta T} \left[ F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {0}, \mathbf {s} _ {0}, s _ {0} ^ {\prime}\right) - \mathbb {E} \left[ F _ {1 / \bar {\rho}} \left(\mathbf {w} _ {T}, \mathbf {s} _ {T}, s _ {T} ^ {\prime}\right) \right] + \frac {3 \eta^ {2} \bar {\rho} M ^ {2} T}{2} \right. \\ \left. + \bar {\rho} \eta C _ {1} \left[ \frac {\Delta_ {0}}{\mu_ {m i n}} + T (R _ {1} + R _ {2} + R _ {3} + R _ {4} + R _ {5}) \right] \right] \\ \leq \frac {2 \Delta}{\eta T} + \left(2 + \frac {n _ {+}}{B _ {1}}\right) \eta \bar {\rho} M ^ {2} + \frac {2 \bar {\rho} C _ {1} \Delta_ {0}}{\mu_ {\text {m i n}} T} + 2 \bar {\rho} C _ {1} \left(R _ {1} + R _ {2} + R _ {3} + R _ {4} + R _ {5}\right) \\ = \mathcal {O} \left(\frac {1}{T} \left(\frac {1}{\eta} + \frac {1}{\mu_ {\text {m i n}}}\right) + \eta + R _ {1} + R _ {2} + R _ {3} + R _ {4} + R _ {5}\right) \\ \end{array}
$$
With MSVR updates for  $v_{i,t}$  and  $u_{i,t}$ , following from Lemma C.5 and Lemma C.6, we have
$$
\mu_ {1} = \frac {B _ {1} \tau_ {1}}{2 n _ {+}}, \quad \mu_ {2} = \frac {B _ {1} \tau_ {1}}{2 n _ {-}}, \quad \mu_ {3} = \frac {B _ {1} \tau_ {2}}{2 n _ {+}}
$$
$$
R _ {1} = \frac {2 \tau_ {1} ^ {1 / 2} \sigma}{B _ {3} ^ {1 / 2}} + \frac {4 n _ {+} C _ {h} M \eta}{B _ {1} \tau_ {1} ^ {1 / 2}}, \quad R _ {2} = \frac {2 \tau_ {1} ^ {1 / 2} \sigma}{B _ {3} ^ {1 / 2}} + \frac {4 n _ {-} C _ {h} M \eta}{B _ {2} \tau_ {1} ^ {1 / 2}}
$$
$$
R _ {3} = \frac {4 \tau_ {1} \sigma^ {2}}{B _ {3}} + \frac {1 6 n _ {+} ^ {2} C _ {h} ^ {2} M ^ {2} \eta^ {2}}{B _ {1} ^ {2} \tau_ {1}}, R _ {4} = \frac {4 \tau_ {1} \sigma^ {2}}{B _ {3}} + \frac {1 6 n _ {-} ^ {2} C _ {h} ^ {2} M ^ {2} \eta^ {2}}{B _ {2} ^ {2} \tau_ {1}}
$$