$$
\begin{array}{l} \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} f _ {i} \left(\frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(h _ {i, j} (\hat {\mathbf {w}} _ {t})\right)\right) - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} f _ {i} \left(u _ {i, t}\right) \\ \stackrel {(a)} {\geq} \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \partial f _ {i} (u _ {i, t}) ^ {\top} (\frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (h _ {i, j} (\hat {\mathbf {w}} _ {t})) - u _ {i, t}) - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \frac {\rho_ {f}}{2} \| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (h _ {i, j} (\hat {\mathbf {w}} _ {t})) - u _ {i, t} \| ^ {2} \\ \geq \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \partial f _ {i} (u _ {i, t}) ^ {\top} \left(\frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(h _ {i, j} (\hat {\mathbf {w}} _ {t})\right) - u _ {i, t}\right) \\ - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \rho_ {f} \| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(h _ {i, j} \left(\hat {\mathbf {w}} _ {t}\right)\right) - \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(v _ {i, j, t}\right) \| ^ {2} - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \rho_ {f} \| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(v _ {i, j, t}\right) - u _ {i, t} \| ^ {2} \\ \geq \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \partial f _ {i} (u _ {i, t}) ^ {\top} \left(\frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(h _ {i, j} (\hat {\mathbf {w}} _ {t})\right) - u _ {i, t}\right) - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} \rho_ {f} C _ {g} ^ {2} \| h _ {i, j} (\hat {\mathbf {w}} _ {t}) - v _ {i, j, t} \| ^ {2} \\ - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \rho_ {f} \| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} \| ^ {2} \\ \geq \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \partial f _ {i} (u _ {i, t}) ^ {\top} \left[ \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} + \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} \partial g _ {i} (v _ {i, j, t}) ^ {\top} \left(h _ {i, j} (\hat {\mathbf {w}} _ {t}) - v _ {i, j, t}\right) \right. \\ \left. - \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} \frac {\rho_ {g}}{2} \left\| h _ {i, j} \left(\hat {\mathbf {w}} _ {t}\right) - v _ {i, j, t} \right\| ^ {2} \right] - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} 2 \rho_ {f} C _ {g} ^ {2} \left\| h _ {i, j} \left(\mathbf {w} _ {t}\right) - v _ {i, j, t} \right\| ^ {2} \\ - 2 \rho_ {f} C _ {g} ^ {2} \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \rho_ {f} \| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} \| ^ {2} \\ \stackrel {(c)} {\geq} \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \partial f _ {i} (u _ {i, t}) ^ {\top} \left[ \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} \right] \\ + \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} \underbrace {\left\langle \partial f _ {i} \left(u _ {i , t}\right) ^ {\top} \partial g _ {i} \left(v _ {i , j , t}\right) ^ {\top} \left(h _ {i , j} \left(\hat {\mathbf {w}} _ {t}\right) - v _ {i , j , t}\right) \right.} _ {A _ {1}} \\ - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} \frac {\rho_ {g} C _ {f}}{2} \| h _ {i, j} (\hat {\mathbf {w}} _ {t}) - v _ {i, j, t} \| ^ {2} - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} 2 \rho_ {f} C _ {g} ^ {2} \| h _ {i, j} (\mathbf {w} _ {t}) - v _ {i, j, t} \| ^ {2} \\ - 2 \rho_ {f} C _ {g} ^ {2} \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \rho_ {f} \| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} \| ^ {2} \\ \geq \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \partial f _ {i} (u _ {i, t}) ^ {\top} \left[ \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} (v _ {i, j, t}) - u _ {i, t} \right] \\ + \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} \underbrace {\left\langle \partial f _ {i} \left(u _ {i , t}\right) ^ {\top} \partial g _ {i} \left(v _ {i , j , t}\right) ^ {\top} \left(h _ {i , j} \left(\hat {\mathbf {w}} _ {t}\right) - v _ {i , j , t}\right) \right.} _ {A _ {1}} \\ - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} \left(2 \rho_ {f} C _ {g} ^ {2} + \rho_ {g} C _ {f}\right) \| h _ {i, j} (\mathbf {w} _ {t}) - v _ {i, j, t} \| ^ {2} \\ - \left(2 \rho_ {f} C _ {g} ^ {2} + \rho_ {g} C _ {f} C _ {h} ^ {2}\right) \| \hat {\mathbf {w}} _ {t} - \mathbf {w} _ {t} \| ^ {2} - \frac {1}{n _ {1}} \sum_ {i \in S _ {1}} \rho_ {f} \| \frac {1}{n _ {2}} \sum_ {j \in S _ {2}} g _ {i} \left(v _ {i, j, t}\right) - u _ {i, t} \| ^ {2} \tag {19} \\ \end{array}
$$