SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \leq \mathbb {E} \left[ F \left(w _ {g} ^ {(r)}\right) \right] - \frac {\eta_ {\ell}}{2} \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) \right\| ^ {2} \\ \left. - \left(\frac {\eta_ {\ell}}{2} - \frac {\beta \eta_ {\ell} ^ {2}}{2}\right) \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) - \frac {1}{M} \sum_ {m \in [ M ]} \alpha_ {m} \sum_ {k = 1} ^ {K} \nabla_ {w _ {g, m} ^ {(r, k - 1)}} f _ {m} \left(w _ {p, m} ^ {(r, k - 1)}\right) - \nabla F \left(w _ {g} ^ {(r)}\right)\right) \right\| ^ {2} \\ + \frac {\eta_ {\ell}}{2} \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) - \frac {1}{M} \sum_ {m \in [ M ]} \alpha_ {m} \sum_ {k = 1} ^ {K} \nabla_ {w _ {g, m} ^ {(r, k - 1)}} f _ {m} \left(w _ {p, m} ^ {(r, k - 1)}\right) \right\| ^ {2} + \frac {\eta_ {\ell} ^ {2} \beta \sigma_ {\ell} ^ {2} K}{2 M} (85) \\ \leq \mathbb {E} \left[ F \left(w _ {g} ^ {(r)}\right) \right] - \left(\frac {3 \eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) \right\| ^ {2} + \frac {\eta_ {\ell} ^ {2} \beta \sigma_ {\ell} ^ {2} K}{2 M} \\ - \left(\frac {\eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) - \frac {1}{M} \sum_ {m \in [ M ]} \alpha_ {m} \sum_ {k = 1} ^ {K} \nabla_ {w _ {g, m} ^ {(r, k - 1)}} f _ {m} \left(w _ {p, m} ^ {(r, k - 1)}\right) \right\| ^ {2} (86) \\ \leq \mathbb {E} \left[ F \left(w _ {g} ^ {(r)}\right) \right] - \left(\frac {3 \eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) \right\| ^ {2} + \frac {\eta_ {\ell} ^ {2} \beta \sigma_ {\ell} ^ {2} K}{2 M} \\ - \left(\frac {\eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) \beta^ {2} K \cdot \frac {1}{M} \sum_ {m \in [ M ]} \alpha_ {m} \sum_ {k = 1} ^ {K} \mathbb {E} \left\| w _ {g} ^ {(r)} - w _ {p, m} ^ {(r, k - 1)} \right\| ^ {2} (87) \\ \end{array}
$$
Using Lemma E.11,
$$
\begin{array}{l} \mathbb {E} \left[ F \left(w _ {g} ^ {(r + 1)}\right) \right] \leq \mathbb {E} \left[ F \left(w _ {g} ^ {(r)}\right) \right] - \left(\frac {3 \eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) \right\| ^ {2} + \frac {\eta_ {\ell} ^ {2} \beta \sigma_ {\ell} ^ {2} K}{2 M} \\ - \left(\frac {\eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) \beta^ {2} K \cdot \frac {1}{M} \sum_ {m \in [ M ]} \alpha_ {m} \sum_ {k = 1} ^ {K} \left(2 0 K ^ {3} \eta_ {\ell} ^ {2} \mathbb {E} | | 1 - \psi_ {g, m} ^ {(r, k)} | | ^ {2} \mathbb {E} | | \nabla f _ {m} (w _ {g} ^ {(r)}) | | ^ {2} \right. \\ \left. + 1 0 K ^ {2} \eta_ {\ell} ^ {2} \sigma_ {\ell} ^ {2} \mathbb {E} | | \psi_ {g, m} ^ {(r, k)} | | ^ {2}\right) \tag {88} \\ \end{array}
$$
Using Assumption E.4 for non-convex case, we get,
$$
\begin{array}{l} \mathbb {E} \left[ F \left(w _ {g} ^ {(r + 1)}\right) \right] \leq \mathbb {E} \left[ F \left(w _ {g} ^ {(r)}\right) \right] - \left(\frac {3 \eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) \right\| ^ {2} + \frac {\eta_ {\ell} ^ {2} \beta \sigma_ {\ell} ^ {2} K}{2 M} \\ - \left(\frac {\eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) 2 0 \beta^ {2} K ^ {4} \eta_ {\ell} ^ {2} (G ^ {2} + B ^ {2} \mathbb {E} | | \nabla F (w _ {g} ^ {(r)}) | | ^ {2}) \mathbb {E} | | 1 - \psi_ {g} ^ {(r)} | | ^ {2} \\ - \left(\frac {\eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) \left(1 0 \beta^ {2} K ^ {3} \eta_ {\ell} ^ {2} \sigma_ {\ell} ^ {2} \mathbb {E} | | \psi_ {g} ^ {(r)} | | ^ {2}\right) \tag {89} \\ \end{array}
$$
Rearranging the terms to put  $\mathbb{E}\left\| \nabla F(w_g^{(r)})\right\|^2$  on left-hand side,
$$
\begin{array}{l} \left(\frac {3 \eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2} - 2 0 K ^ {4} \beta^ {2} \eta_ {\ell} ^ {2} B ^ {2} \mathbf {q} _ {1} ^ {2} \left(\frac {\eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right)\right) \mathbb {E} \left\| \nabla F (w _ {g} ^ {(r)}) \right\| ^ {2} \leq \mathbb {E} \left[ F (w _ {g} ^ {(r)}) \right] \\ - \mathbb {E} \left[ F \left(w _ {g} ^ {(r + 1)}\right) \right] - 2 0 \mathbf {q} _ {1} ^ {2} K ^ {4} \beta^ {2} \eta_ {\ell} ^ {2} G ^ {2} \left(\frac {\eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) \\ - 1 0 \mathbf {q} _ {0} ^ {2} K ^ {3} \beta^ {2} \eta_ {\ell} ^ {2} \sigma_ {\ell} ^ {2} \left(\frac {\eta_ {\ell}}{2} - \beta \eta_ {\ell} ^ {2}\right) + \frac {\eta_ {\ell} ^ {2} \beta \sigma_ {\ell} ^ {2} K}{2 M} \tag {90} \\ \end{array}
$$
Assuming  $10K^4\beta^2\eta_\ell^3 B^2 \leq \frac{\eta_\ell}{2} \Rightarrow \eta_\ell \leq \frac{1}{2\sqrt{5}\beta B K^2}$  and  $20K^4\beta^3\eta_\ell^4 B^2 \leq \frac{\eta_\ell}{2} \Rightarrow \eta_\ell \leq \frac{1}{\sqrt[3]{40K^4\beta^3B^2}}$ ,
$$
\begin{array}{l} \left(\frac {\eta_ {\ell}}{2}\right) \mathbf {q} _ {0} ^ {2} \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) \right\| ^ {2} \leq \mathbb {E} \left[ F \left(w _ {g} ^ {(r)}\right) \right] - \mathbb {E} \left[ F \left(w _ {g} ^ {(r + 1)}\right) \right] + \frac {\eta_ {\ell} ^ {2} \beta \sigma_ {\ell} ^ {2} K}{2 M} \\ + 2 0 \mathbf {q} _ {1} ^ {2} K ^ {4} \beta^ {2} \eta_ {\ell} ^ {2} G ^ {2} \left(\frac {2 \beta \eta_ {\ell} ^ {2} - \eta_ {\ell}}{2}\right) + 1 0 \mathbf {q} _ {0} ^ {2} K ^ {3} \beta^ {2} \eta_ {\ell} ^ {2} \sigma_ {\ell} ^ {2} \left(\beta \eta_ {\ell} ^ {2} - \frac {\eta_ {\ell}}{2}\right) \tag {91} \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) \right\| ^ {2} \leq \frac {2}{\eta_ {\ell} \mathbf {q} _ {0} ^ {2}} \left[ \mathbb {E} \left[ F \left(w _ {g} ^ {(r)}\right) \right] - \mathbb {E} \left[ F \left(w _ {g} ^ {(r + 1)}\right) \right] \right] + \frac {\eta_ {\ell} \beta \sigma_ {\ell} ^ {2} K}{M \mathbf {q} _ {0} ^ {2}} \\ + 4 0 \frac {\mathbf {q} _ {1} ^ {2}}{\mathbf {q} _ {0} ^ {2}} K ^ {4} \beta^ {2} \eta_ {\ell} G ^ {2} \left(\frac {2 \beta \eta_ {\ell} ^ {2} - \eta_ {\ell}}{2}\right) + 2 0 K ^ {3} \beta^ {2} \eta_ {\ell} \sigma_ {\ell} ^ {2} \left(\beta \eta_ {\ell} ^ {2} - \frac {\eta_ {\ell}}{2}\right) \tag {92} \\ \end{array}
$$
Taking average over all the  $R$  rounds,
$$
\begin{array}{l} \frac {1}{R} \sum_ {r = 1} ^ {R} \mathbb {E} \left\| \nabla F \left(w _ {g} ^ {(r)}\right) \right\| ^ {2} \leq \frac {2}{\eta_ {\ell} \mathbf {q} _ {0} ^ {2} R} \left[ \mathbb {E} \left[ F \left(w _ {g} ^ {(1)}\right) \right] - \mathbb {E} \left[ F \left(w _ {g} ^ {(R + 1)}\right) \right] \right] + \frac {\eta_ {\ell} \beta \sigma_ {\ell} ^ {2} K}{M \mathbf {q} _ {0} ^ {2}} \\ + 4 0 \frac {\mathbf {q} _ {1} ^ {2}}{\mathbf {q} _ {0} ^ {2}} K ^ {4} \beta^ {2} \eta_ {\ell} G ^ {2} \left(\frac {2 \beta \eta_ {\ell} ^ {2} - \eta_ {\ell}}{2}\right) + 2 0 K ^ {3} \beta^ {2} \eta_ {\ell} \sigma_ {\ell} ^ {2} \left(\beta \eta_ {\ell} ^ {2} - \frac {\eta_ {\ell}}{2}\right) \tag {93} \\ \end{array}
$$
E.5 Convergence Proof for the Personalized Model: Convex (Strong and General) Cases
Lemma E.13 (Local progress of the personalized model). If  $m^{th}$  client's objective function  $f_{m}$  satisfies Assumptions E.1, E.2, E.3, and E.4 and conditioning on  $\eta_{\ell} \leq \frac{1}{\beta \sqrt{6K}}$  in Algorithm 2, the following are satisfied:
$$
\begin{array}{l} \mathbb {E} \| w _ {p, m} ^ {(r, K)} - \tilde {w} _ {p, m} ^ {(r, 0)} \| ^ {2} \leq 1 8 K ^ {2} \eta_ {\ell} ^ {2} \mathbb {E} \| \nabla f _ {m} (\tilde {w} _ {p, m} ^ {(r, 0)}) \| ^ {2} + 1 0 8 K ^ {4} \eta_ {\ell} ^ {4} \mathbb {E} \| \nabla f _ {m} (w _ {g} ^ {(r)}) \| ^ {2} + 1 2 6 K ^ {3} \eta_ {\ell} ^ {4} \sigma_ {\ell} ^ {2} \\ + 9 K ^ {2} \eta_ {\ell} ^ {2} \mathbb {E} \left\| \psi_ {g, m} ^ {(r, K)} \right\| ^ {2} + 1 4 4 K ^ {5} \eta_ {\ell} ^ {4} \mathbb {E} | | \psi_ {g, m} ^ {(r, K)} | | ^ {2} \mathbb {E} | | \nabla f _ {m} (w _ {g} ^ {(r)}) | | ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} \| w _ {p, m} ^ {(r, K)} - \tilde {w} _ {p, m} ^ {(r, 0)} \| ^ {2} \leq \mathbb {E} \| \psi_ {g, m} ^ {(r, K + 1)} w _ {g, m} ^ {(r, K)} + (1 - \psi_ {g, m} ^ {(r, K + 1)}) w _ {\ell , m} ^ {(r, K)} \\ - \psi_ {g, m} ^ {(r, 0)} w _ {g, m} ^ {(r, 0)} - (1 - \psi_ {g, m} ^ {(r, 0)}) w _ {\ell , m} ^ {(r, K)} \| ^ {2} (94) \\ = \mathbb {E} \left\| \left(\psi_ {g, m} ^ {(r, K)} - \eta_ {\ell} \nabla_ {\psi_ {g, m} ^ {(r, K)}} f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, K)}\right)\right) \left(w _ {g, m} ^ {(r, K - 1)} - \eta_ {\ell} \nabla_ {w _ {g, m} ^ {(r, K - 1)}} f _ {m} \left(w _ {p, m} ^ {(r, K - 1)}\right)\right) \right. \\ \left. + \left(1 - \psi_ {g, m} ^ {(r, K)} + \eta_ {\ell} \nabla_ {\psi_ {g, m} ^ {(r, K)}} f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, K)}\right)\right) w _ {\ell , m} ^ {(r, K)} - \psi_ {g, m} ^ {(r, 0)} w _ {g, m} ^ {(r, 0)} - \left(1 - \psi_ {g, m} ^ {(r, 0)}\right) w _ {\ell , m} ^ {(r, K)} \right\| ^ {2} (95) \\ = \mathbb {E} | | \psi_ {g, m} ^ {(r, K)} w _ {g, m} ^ {(r, K - 1)} - \psi_ {g, m} ^ {(r, K)} \eta_ {\ell} \nabla_ {w _ {g, m} ^ {(r, K - 1)}} f _ {m} (w _ {p, m} ^ {(r, K - 1)}) - w _ {g, m} ^ {(r, K - 1)} \eta_ {\ell} \nabla_ {\psi_ {g, m} ^ {(r, K)}} f _ {m} (\tilde {w} _ {p, m} ^ {(r, K)}) \\ + \eta_ {\ell} ^ {2} \nabla_ {\psi_ {g, m} ^ {(r, K)}} f _ {m} (\tilde {w} _ {p, m} ^ {(r, K)}) \nabla_ {w _ {g, m} ^ {(r, K - 1)}} f _ {m} (w _ {p, m} ^ {(r, K - 1)}) + \left(1 - \psi_ {g, m} ^ {(r, K)}\right) w _ {\ell , m} ^ {(r, K)} \\ + w _ {\ell , m} ^ {(r, K)} \eta_ {\ell} \nabla_ {\psi_ {g, m} ^ {(r, K)}} f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, K)}\right) - \psi_ {g, m} ^ {(r, 0)} w _ {g, m} ^ {(r, 0)} - \left(1 - \psi_ {g, m} ^ {(r, 0)}\right) w _ {\ell , m} ^ {(r, K)} \| ^ {2} (96) \\ \end{array}
$$
Using the convexity of  $f_{m}$
$$
\begin{array}{l} \nabla_ {w _ {g, m} ^ {(r, k)}} f _ {m} \left(w _ {p, m} ^ {(r, k)}\right) = \nabla_ {w _ {g, m} ^ {(r, k)}} f _ {m} \left(\psi_ {g, m} ^ {(r, k + 1)} w _ {g, m} ^ {(r, k)} + \left(1 - \psi_ {g, m} ^ {(r, k + 1)}\right) w _ {\ell , m} ^ {(r, K)}\right) (97) \\ \leq \psi_ {g, m} ^ {(r, k + 1)} \nabla f _ {m} \left(w _ {g, m} ^ {(r, k)}\right) (98) \\ \end{array}
$$
$$
\begin{array}{l} \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, k)}\right) = \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} \left(\psi_ {g, m} ^ {(r, k)} \left[ w _ {g, m} ^ {(r, k)} - w _ {\ell , m} ^ {(r, K)} \right] - w _ {\ell , m} ^ {(r, K)}\right) (99) \\ \leq \left(w _ {g, m} ^ {(r, k)} - w _ {\ell , m} ^ {(r, K)}\right) \nabla f _ {m} \left(\psi_ {g, m} ^ {(r, k)}\right) (100) \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} | | w _ {p, m} ^ {(r, K)} - \tilde {w} _ {p, m} ^ {(r, 0)} | | ^ {2} \leq \mathbb {E} | | \psi_ {g, m} ^ {(r, K)} w _ {g, m} ^ {(r, K - 1)} + \left(1 - \psi_ {g, m} ^ {(r, K)}\right) w _ {\ell , m} ^ {(r, K)} - \psi_ {g, m} ^ {(r, 0)} w _ {g, m} ^ {(r, 0)} - (1 - \psi_ {g, m} ^ {(r, 0)}) w _ {\ell , m} ^ {(r, K)} \\ \left. - \eta_ {\ell} \left(\psi_ {g, m} ^ {(r, K)}\right) ^ {2} \nabla f _ {m} \left(w _ {g, m} ^ {(r, K - 1)}\right) + \eta_ {\ell} \left(w _ {\ell , m} ^ {(r, K)} - w _ {g, m} ^ {(r, K - 1)}\right) ^ {2} \nabla f _ {m} \left(w _ {g, m} ^ {(r, K)}\right) \right\| ^ {2} (101) \\ \leq \left(1 + \frac {1}{K - 1}\right) \mathbb {E} \left| \left| w _ {p, m} ^ {(r, K - 1)} - \tilde {w} _ {p, m} ^ {(r, 0)} \right| \right| ^ {2} + 3 K \eta_ {\ell} ^ {2} \mathbb {E} \left\| \nabla f _ {m} \left(w _ {p, m} ^ {(r, K - 1)}\right) \right\| ^ {2} \\ + 3 K \eta_ {\ell} ^ {2} \mathbb {E} \left\| w _ {\ell , m} ^ {(r, K)} - w _ {g, m} ^ {(r, K - 1)} \right\| ^ {2} + 3 K \eta_ {\ell} ^ {2} \mathbb {E} \left\| \psi_ {g, m} ^ {(r, K)} \right\| ^ {2} (102) \\ \end{array}
$$
$$
\begin{array}{l} \leq \left(1 + \frac {1}{K - 1}\right) \mathbb {E} | | w _ {p, m} ^ {(r, K - 1)} - \tilde {w} _ {p, m} ^ {(r, 0)} | | ^ {2} + 3 K \eta_ {\ell} ^ {2} \mathbb {E} \left\| \nabla f _ {m} \left(w _ {p, m} ^ {(r, K - 1)}\right) - \nabla f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, 0)}\right) + \nabla f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, 0)}\right) \right\| ^ {2} \\ + 6 K \eta_ {\ell} ^ {2} \mathbb {E} \| w _ {\ell , m} ^ {(r, K)} - w _ {g, m} ^ {(r, 0)} \| ^ {2} + 6 K \eta_ {\ell} ^ {2} \mathbb {E} \| w _ {g, m} ^ {(r, 0)} - w _ {g, m} ^ {(r, K - 1)} \| ^ {2} + 3 K \eta_ {\ell} ^ {2} \mathbb {E} \| \psi_ {g, m} ^ {(r, K)} \| ^ {2} \tag {103} \\ \end{array}
$$
Using Lemma E.6 and E.7, and smoothness property,
$$
\begin{array}{l} \leq \left(1 + \frac {1}{K - 1} + 6 K \eta_ {\ell} ^ {2} \beta^ {2}\right) \mathbb {E} \left| \left| w _ {p, m} ^ {(r, K - 1)} - \tilde {w} _ {p, m} ^ {(r, 0)} \right| \right| ^ {2} + 6 K \eta_ {\ell} ^ {2} \mathbb {E} \left\| \nabla f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, 0)}\right) \right\| ^ {2} \\ + 6 K \eta_ {\ell} ^ {2} \left(6 K ^ {2} \eta_ {\ell} ^ {2} \mathbb {E} | | \nabla f _ {m} \left(w _ {\ell , m} ^ {(r, 0)}\right) | | ^ {2} + 3 K \eta_ {\ell} ^ {2} \sigma_ {\ell} ^ {2}\right) + 3 K \eta_ {\ell} ^ {2} \mathbb {E} \left| \left| \psi_ {g, m} ^ {(r, K)} \right| \right| ^ {2} \\ + 6 K \eta_ {\ell} ^ {2} \left(8 K ^ {- 3} \eta_ {\ell} ^ {2} \mathbb {E} | | \psi_ {g, m} ^ {(r, K)} | | ^ {2} \mathbb {E} | | \nabla f _ {m} \left(w _ {g} ^ {(r)}\right) | | ^ {2} + 4 K \eta_ {\ell} ^ {2} \sigma_ {\ell} ^ {2}\right) \tag {104} \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} \| w _ {p, m} ^ {(r, K)} - \tilde {w} _ {p, m} ^ {(r, 0)} \| ^ {2} \leq \sum_ {i = 1} ^ {K} \left(6 K \eta_ {\ell} ^ {2} \mathbb {E} \| \nabla f _ {m} (\tilde {w} _ {p, m} ^ {(r, 0)}) \| ^ {2} + 3 6 K ^ {3} \eta_ {\ell} ^ {4} \mathbb {E} \| \nabla f _ {m} (w _ {g} ^ {(r)}) \| ^ {2} + 4 2 K ^ {2} \eta_ {\ell} ^ {4} \sigma_ {\ell} ^ {2}\right) \\ \left. + 3 K \eta_ {\ell} ^ {2} \mathbb {E} \left\| \psi_ {g, m} ^ {(r, K)} \right\| ^ {2} + 4 8 K ^ {4} \eta_ {\ell} ^ {4} \mathbb {E} \left\| \psi_ {g, m} ^ {(r, K)} \right\| ^ {2} \mathbb {E} \left\| \nabla f _ {m} \left(w _ {g} ^ {(r)}\right) \right\| ^ {2}\right) \left(1 + \frac {1}{K - 1} + 6 K \eta_ {\ell} ^ {2} \beta^ {2}\right) ^ {i} \tag {105} \\ \end{array}
$$
Assuming  $6K\eta_{\ell}^{2}\beta^{2}\leq 1\Rightarrow \eta_{\ell}\leq \frac{1}{\beta\sqrt{6K}}$
$$
\begin{array}{l} \mathbb {E} \| w _ {p, m} ^ {(r, K)} - \tilde {w} _ {p, m} ^ {(r, 0)} \| ^ {2} \leq 3 K \left(6 K \eta_ {\ell} ^ {2} \mathbb {E} \left\| \nabla f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, 0)}\right) \right\| ^ {2} + 3 6 K ^ {3} \eta_ {\ell} ^ {4} \mathbb {E} \| \nabla f _ {m} \left(w _ {g} ^ {(r)}\right) \| ^ {2} + 4 2 K ^ {2} \eta_ {\ell} ^ {4} \sigma_ {\ell} ^ {2} \right. \\ \left. + 3 K \eta_ {\ell} ^ {2} \mathbb {E} \left\| \psi_ {g, m} ^ {(r, K)} \right\| ^ {2} + 4 8 K ^ {4} \eta_ {\ell} ^ {4} \mathbb {E} \left\| \psi_ {g, m} ^ {(r, K)} \right\| ^ {2} \mathbb {E} \| \nabla f _ {m} \left(w _ {g} ^ {(r)}\right) \| ^ {2}\right) (106) \\ = 1 8 K ^ {2} \eta_ {\ell} ^ {2} \mathbb {E} \left\| \nabla f _ {m} (\tilde {w} _ {p, m} ^ {(r, 0)}) \right\| ^ {2} + 1 0 8 K ^ {4} \eta_ {\ell} ^ {4} \mathbb {E} \| \nabla f _ {m} (w _ {g} ^ {(r)}) \| ^ {2} + 1 2 6 K ^ {3} \eta_ {\ell} ^ {4} \sigma_ {\ell} ^ {2} \\ + 9 K ^ {2} \eta_ {\ell} ^ {2} \mathbb {E} \left\| \psi_ {g, m} ^ {(r, K)} \right\| ^ {2} + 1 4 4 K ^ {5} \eta_ {\ell} ^ {4} \mathbb {E} \| \psi_ {g, m} ^ {(r, K)} \| ^ {2} \mathbb {E} \| \nabla f _ {m} \left(w _ {g} ^ {(r)}\right) \| ^ {2} (107) \\ \end{array}
$$
Lemma E.14 (Deviation of local parameters from the aggregated global parameters). If  $m^{th}$  client's objective function  $f_{m}$  satisfies Assumptions E.3, E.4, in Algorithm 2, the following is satisfied:
$$
\begin{array}{l} \mathbb {E} | | \tilde {w} _ {p, m} ^ {(r + 1, 0)} - w _ {p, m} ^ {(r, K)} | | ^ {2} \leq 1 8 \left(K \sigma_ {\ell} ^ {2} \eta_ {\ell} ^ {2} + \left(\delta_ {m} ^ {\psi} + \frac {\delta^ {\psi}}{M}\right) K ^ {2} \eta_ {\ell} ^ {2}\right) \left(K \sigma_ {\ell} ^ {2} \eta_ {\ell} ^ {2} + \left(\delta_ {m} ^ {w _ {g}} + \frac {\delta^ {w _ {g}}}{M}\right) K ^ {2} \eta_ {\ell} ^ {2}\right) \\ + 6 \left(1 + \eta_ {\ell} ^ {2} K ^ {2} \beta^ {4}\right) \eta_ {\ell} ^ {2} K ^ {2} \left(K \sigma_ {\ell} ^ {2} \eta_ {\ell} ^ {2} + \left(\delta_ {m} ^ {\psi} + \frac {\delta^ {\psi}}{M}\right) K ^ {2} \eta_ {\ell} ^ {2}\right) \left(G ^ {2} + B ^ {2} \mathbb {E} | | \nabla F (w _ {g} ^ {(r)}) | | ^ {2}\right) \\ \end{array}
$$
Proof. Stating the aggregate rule from Algorithm 2, Lines 12,19 and 20,
$$
\begin{array}{l} \mathbb {E} \| \tilde {w} _ {p, m} ^ {(r + 1, 0)} - w _ {p, m} ^ {(r, K)} \| ^ {2} = \mathbb {E} \Big \| \frac {1}{M} \sum_ {c \in [ M ]} \psi_ {g, c} ^ {(r, K)} \frac {1}{M} \sum_ {c \in [ M ]} w _ {g, c} ^ {(r, K)} + \left(1 - \frac {1}{M} \sum_ {c \in [ M ]} \psi_ {g, c} ^ {(r, K)}\right) w _ {\ell , m} ^ {(r + 1, K)} \\ \left. - \psi_ {g, m} ^ {(r, K)} w _ {g, m} ^ {(r, K)} - \left(1 - \psi_ {g, m} ^ {(r, K)}\right) w _ {\ell , m} ^ {(r, K)} \right\| ^ {2} (108) \\ \leq 2 \mathbb {E} \left\| \frac {1}{M} \sum_ {c \in [ M ]} \psi_ {g, c} ^ {(r, K)} \frac {1}{M} \sum_ {c \in [ M ]} w _ {g, c} ^ {(r, K)} - \psi_ {g, m} ^ {(r, K)} w _ {g, m} ^ {(r, K)} \right\| ^ {2} \\ + 2 \mathbb {E} \| \left(1 - \frac {1}{M} \sum_ {c \in [ M ]} \psi_ {g, c} ^ {(r, K)}\right) w _ {\ell , m} ^ {(r + 1, K)} - (1 - \psi_ {g, m} ^ {(r, K)}) w _ {\ell , m} ^ {(r, K)} \| ^ {2} (109) \\ \leq 2 \mathbb {E} \left\| \left(\frac {1}{M} \sum_ {c \in [ M ]} \psi_ {g, c} ^ {(r, K)} - \psi_ {g, m} ^ {(r, K)}\right) \left(\frac {1}{M} \sum_ {c \in [ M ]} w _ {g, c} ^ {(r, K)} - w _ {g, m} ^ {(r, K)}\right) \right\| ^ {2} \\ + 2 \mathbb {E} \left\| \left(\psi_ {g, m} ^ {(r, K)} - \frac {1}{M} \sum_ {c \in [ M ]} \psi_ {g, c} ^ {(r, K)}\right) \left(w _ {\ell , m} ^ {(r + 1, K)} - w _ {\ell , m} ^ {(r, K)}\right) \right\| ^ {2} (110) \\ \end{array}
$$
Using Lemma 8 from [29] and Lemma E.17,
$$
\begin{array}{l} \mathbb {E} | | \tilde {w} _ {p, m} ^ {(r + 1, 0)} - w _ {p, m} ^ {(r, K)} | | ^ {2} \leq 1 8 \left(K \sigma_ {\ell} ^ {2} \eta_ {\ell} ^ {2} + \left(\delta_ {m} ^ {\psi} + \frac {\delta^ {\psi}}{M}\right) K ^ {2} \eta_ {\ell} ^ {2}\right) \left(K \sigma_ {\ell} ^ {2} \eta_ {\ell} ^ {2} + \left(\delta_ {m} ^ {w _ {g}} + \frac {\delta^ {w _ {g}}}{M}\right) K ^ {2} \eta_ {\ell} ^ {2}\right) \\ + 6 \left(1 + \eta_ {\ell} ^ {2} K ^ {2} \beta^ {4}\right) \eta_ {\ell} ^ {2} K ^ {2} \left(K \sigma_ {\ell} ^ {2} \eta_ {\ell} ^ {2} + \left(\delta_ {m} ^ {\psi} + \frac {\delta^ {\psi}}{M}\right) K ^ {2} \eta_ {\ell} ^ {2}\right) \\ \left(G ^ {2} + B ^ {2} \mathbb {E} \| \nabla F \left(w _ {g} ^ {(r)}\right) \| ^ {2}\right) \tag {111} \\ \end{array}
$$
Lemma E.15 (One epoch progress of the personalized model). If  $m^{th}$  client's objective function  $f_{m}$  satisfies Assumptions E.1, E.2, E.3, and E.4 in Algorithm 2, the following are satisfied:
$$
\mathbb {E} \| w _ {p, m} ^ {(r, k + 1)} - w _ {p, m} ^ {(r, k)} \| ^ {2} \leq 3 \eta_ {\ell} ^ {2} \mathbb {E} \big \| \nabla f _ {m} (w _ {p, m} ^ {(r, k)}) \big \| ^ {2} + 3 \eta_ {\ell} ^ {2} \mathbb {E} \big \| w _ {\ell , m} ^ {(r, K)} - w _ {g, m} ^ {(r, k)} \big \| ^ {2} + 3 \eta_ {\ell} ^ {2} \mathbb {E} \| \psi_ {g, m} ^ {(r, k)} \| ^ {2}
$$
$$
\begin{array}{l} \mathbb {E} \left| \left| w _ {p, m} ^ {(r, K)} - w _ {p, m} ^ {(r, k)} \right| \right| ^ {2} \leq 6 \beta \eta_ {\ell} ^ {2} \left(\mathbb {E} \left[ f _ {m} \left(w _ {p, m} ^ {(r, K)}\right) \right] - f \left(w _ {p, m} ^ {*}\right)\right) + 3 \eta_ {\ell} ^ {2} K \sum_ {i = k} ^ {K} \mathbb {E} \left| \left| \psi_ {g, m} ^ {(r, i)} \right| \right| ^ {2} \\ + 3 6 K ^ {3} \eta_ {\ell} ^ {4} \mathbb {E} | | \nabla f _ {m} (w _ {g} ^ {(r)}) | | ^ {2} + 4 0 K ^ {2} \eta_ {\ell} ^ {4} \sigma_ {\ell} ^ {2} \\ + 4 8 K ^ {4} \eta_ {\ell} ^ {4} \mathbb {E} | | \nabla f _ {m} \left(w _ {g} ^ {(r)}\right) | | ^ {2} \sum_ {i = k} ^ {K} \mathbb {E} | | \psi_ {g, m} ^ {(r, i)} | | ^ {2} \\ \end{array}
$$
$$
\begin{array}{l} \mathbb {E} \| w _ {p, m} ^ {(r, k + 1)} - w _ {p, m} ^ {(r, k)} \| ^ {2} = \mathbb {E} \| \psi_ {g, m} ^ {(r, k + 1)} w _ {g, m} ^ {(r, k + 1)} \\ + \left(1 - \psi_ {g, m} ^ {(r, k + 1)}\right) w _ {\ell , m} ^ {(r, K)} - \psi_ {g, m} ^ {(r, k)} w _ {g, m} ^ {(r, k)} - \left(1 - \psi_ {g, m} ^ {(r, k)}\right) w _ {\ell , m} ^ {(r, K)} \| ^ {2} (112) \\ = \mathbb {E} \left\| \left(\psi_ {g, m} ^ {(r, k)} - \eta_ {\ell} \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} (\tilde {w} _ {p, m} ^ {(r, k)})\right) \left(w _ {g, m} ^ {(r, k)} - \eta_ {\ell} \nabla_ {w _ {g, m} ^ {(r, k)}} f _ {m} (w _ {p, m} ^ {(r, k)})\right) \right. \\ \left. + \left(1 - \psi_ {g, m} ^ {(r, k)} + \eta_ {\ell} \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} (\tilde {w} _ {p, m} ^ {(r, k)})\right) w _ {\ell , m} ^ {(r, K)} \right. \\ \left. - \psi_ {g, m} ^ {(r, k)} w _ {g, m} ^ {(r, k)} - \left(1 - \psi_ {g, m} ^ {(r, k)}\right) w _ {\ell , m} ^ {(r, K)} \right\| ^ {2} (113) \\ = \mathbb {E} \left\| \psi_ {g, m} ^ {(r, k)} w _ {g, m} ^ {(r, k)} - \eta_ {\ell} w _ {g, m} ^ {(r, k)} \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, k)}\right) - \eta_ {\ell} \psi_ {g, m} ^ {(r, k)} \nabla_ {w _ {g, m} ^ {(r, k)}} f _ {m} \left(w _ {p, m} ^ {(r, k)}\right) \right. \\ + \eta_ {\ell} ^ {2} \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} (\tilde {w} _ {p, m} ^ {(r, k)}) \nabla_ {w _ {g, m} ^ {(r, k)}} f _ {m} (w _ {p, m} ^ {(r, k)}) + (1 - \psi_ {g, m} ^ {(r, k)}) w _ {\ell , m} ^ {(r, K)} \\ \left. + \eta_ {\ell} w _ {\ell , m} ^ {(r, K)} \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, k)}\right) - \psi_ {g, m} ^ {(r, k)} w _ {g, m} ^ {(r, k)} - \left(1 - \psi_ {g, m} ^ {(r, k)}\right) w _ {\ell , m} ^ {(r, K)} \right\| ^ {2} (114) \\ = \mathbb {E} \left\| \eta_ {\ell} \left(w _ {\ell , m} ^ {(r, K)} - w _ {g, m} ^ {(r, k)}\right) \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, k)}\right) \right. \\ \left. - \eta_ {\ell} \left(\psi_ {g, m} ^ {(r, k + 1)}\right) \nabla_ {w _ {g, m} ^ {(r, k)}} f _ {m} \left(w _ {p, m} ^ {(r, k)}\right) \right\| ^ {2} (115) \\ \end{array}
$$
Using,
$$
\begin{array}{l} \nabla_ {w _ {g, m} ^ {(r, k)}} f _ {m} \left(w _ {p, m} ^ {(r, k)}\right) = \nabla_ {w _ {g, m} ^ {(r, k)}} f _ {m} \left(\psi_ {g, m} ^ {(r, k + 1)} w _ {g, m} ^ {(r, k)} + \left(1 - \psi_ {g, m} ^ {(r, k + 1)}\right) w _ {\ell , m} ^ {(r, K)}\right) (116) \\ \leq \psi_ {g, m} ^ {(r, k + 1)} \nabla f _ {m} \left(w _ {g, m} ^ {(r, k)}\right) (117) \\ \end{array}
$$
$$
\begin{array}{l} \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} \left(\tilde {w} _ {p, m} ^ {(r, k)}\right) = \nabla_ {\psi_ {g, m} ^ {(r, k)}} f _ {m} \left(\psi_ {g, m} ^ {(r, k)} \left[ w _ {g, m} ^ {(r, k)} - w _ {\ell , m} ^ {(r, K)} \right] - w _ {\ell , m} ^ {(r, K)}\right) (118) \\ \leq \left[ w _ {g, m} ^ {(r, k)} - w _ {\ell , m} ^ {(r, K)} \right] \nabla f _ {m} \left(\psi_ {g, m} ^ {(r, k)}\right), (119) \\ \end{array}
$$
$$
\begin{array}{l} \leq \eta_ {\ell} ^ {2} \mathbb {E} \| - \left(w _ {\ell , m} ^ {(r, K)} - w _ {g, m} ^ {(r, k)}\right) ^ {2} \nabla f _ {m} \left(\psi_ {g, m} ^ {(r, k)}\right) - \left(\psi_ {g, m} ^ {(r, k + 1)}\right) ^ {2} \nabla f _ {m} \left(w _ {g, m} ^ {(r, k)}\right) \| ^ {2} (120) \\ \leq \eta_ {\ell} ^ {2} \mathbb {E} \| \nabla f _ {m} \left(w _ {p, m} ^ {(r, k)}\right) + \left(w _ {\ell , m} ^ {(r, K)} - w _ {g, m} ^ {(r, k)}\right) + \psi_ {g, m} ^ {(r, k + 1)} \| ^ {2} (121) \\ \leq 3 \eta_ {\ell} ^ {2} \mathbb {E} \left\| \nabla f _ {m} \left(w _ {p, m} ^ {(r, k)}\right) \right\| ^ {2} + 3 \eta_ {\ell} ^ {2} \mathbb {E} \left\| w _ {\ell , m} ^ {(r, K)} - w _ {g, m} ^ {(r, k)} \right\| ^ {2} + 3 \eta_ {\ell} ^ {2} \mathbb {E} \left\| \psi_ {g, m} ^ {(r, k + 1)} \right\| ^ {2} (122) \\ \end{array}
$$
From Lemmas E.6 and E.7.
Summing over  $i = k$  to  $K$ ,