$$
\begin{array}{l} \mathbb {E} _ {t} \left[ F _ {1 / \bar {\rho}} (\mathbf {w} _ {t + 1}) \right] \\ \leq F _ {1 / \bar {\rho}} (\mathbf {w} _ {t}) + \frac {\eta^ {2} \bar {\rho} M ^ {2}}{2} + \bar {\rho} \eta \left\{\frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \left[ f _ {i} \left(\frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(h _ {i, j} (\hat {\mathbf {w}} _ {t})\right)\right) - f _ {i} \left(u _ {i, t}\right) \right. \right. \\ + C _ {f} \left\| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(v _ {i, j, t}\right) - u _ {i, t} \right\| + \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} C _ {f} C _ {g} \left\| h _ {i, j} \left(\mathbf {w} _ {t}\right) - v _ {i, j, t} \right\| \\ + \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} \left(2 \rho_ {f} C _ {g} ^ {2} + \rho_ {g} C _ {f}\right) \| h _ {i, j} (\mathbf {w} _ {t}) - v _ {i, j, t} \| ^ {2} \\ \left. + \left(2 \rho_ {f} C _ {g} ^ {2} + \rho_ {g} C _ {f} C _ {h} ^ {2} + \frac {C _ {f} C _ {g} L _ {h}}{2}\right) \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} + \rho_ {f} \left\| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} \right\| ^ {2} \right] \Biggr \} \\ \leq F _ {1 / \bar {\rho}} (\mathbf {w} _ {t}) + \frac {\eta^ {2} \bar {\rho} M ^ {2}}{2} + \bar {\rho} \eta \left\{\frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \left[ F _ {i} (\hat {\mathbf {w}} _ {t}) - F _ {i} (\mathbf {w} _ {t}) + F _ {i} (\mathbf {w} _ {t}) - f _ {i} \left(\frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t})\right) \right. \right. \\ + f _ {i} \left(\frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(v _ {i, j, t}\right)\right) - f _ {i} \left(u _ {i, t}\right) + C _ {f} \left\| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(v _ {i, j, t}\right) - u _ {i, t} \right\| \\ + \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} C _ {f} C _ {g} \| h _ {i, j} (\mathbf {w} _ {t}) - v _ {i, j, t} \| + \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} (2 \rho_ {f} C _ {g} ^ {2} + \rho_ {g} C _ {f}) \| h _ {i, j} (\mathbf {w} _ {t}) - v _ {i, j, t} \| ^ {2} \\ \left. + (2 \rho_ {f} C _ {g} ^ {2} + \rho_ {g} C _ {f} C _ {h} ^ {2} + \frac {C _ {f} C _ {g} L _ {h}}{2}) \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} + \rho_ {f} \left\| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} \right\| ^ {2} \right] \Biggr \} \\ \stackrel {(a)} {\leq} F _ {1 / \bar {\rho}} (\mathbf {w} _ {t}) + \frac {\eta^ {2} \bar {\rho} M ^ {2}}{2} + \bar {\rho} \eta \left\{\frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \left[ F _ {i} (\hat {\mathbf {w}} _ {t}) - F _ {i} (\mathbf {w} _ {t}) + 2 C _ {f} \right\| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} \right\| \\ + \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} 2 C _ {f} C _ {g} \| h _ {i, j} (\mathbf {w} _ {t}) - v _ {i, j, t} \| + \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} (2 \rho_ {f} C _ {g} ^ {2} + \rho_ {g} C _ {f}) \| h _ {i, j} (\mathbf {w} _ {t}) - v _ {i, j, t} \| ^ {2} \\ \left. + \left(2 \rho_ {f} C _ {g} ^ {2} + \rho_ {g} C _ {f} C _ {h} ^ {2} + \frac {C _ {f} C _ {g} L _ {h}}{2}\right) \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} + \rho_ {f} \left\| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} \right\| ^ {2} \right] \Biggr \} \\ \end{array}
$$