SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\sigma_ {\text {w e a k}} \triangleq \frac {1 + \tau \left(2 \ell_ {g} + \ell_ {g} ^ {\prime}\right) \sigma_ {2}}{2 \tau (p - L)}, \quad \text {a n d} \quad \sigma_ {2} \triangleq \frac {p + L}{p - L}. \tag {80}
$$
Proof. First, let
$$
\begin{array}{l} \lambda_ {+} (v) = P _ {+} \left[ \lambda + \tau \nabla_ {\lambda} K \left(x ^ {*} (v; \lambda), y ^ {*} (v; \lambda), v; \lambda\right) \right], (81a) \\ \lambda (v) \in \arg \max  _ {0 \leq \lambda \leq \Lambda} K (\bar {x} ^ {*} (v), \bar {y} ^ {*} (v), v; \lambda). (81b) \\ \end{array}
$$
which give the desired result.
Based on the strong convexity of  $K(x, \cdot, w, z; \lambda)$  and  $K(\cdot, y, w, z; \lambda)$ , we have
$$
\begin{array}{l} K \left(\bar {x} ^ {*} (v), \bar {y} ^ {*} (v), v; \lambda_ {+} (v)\right) - K \left(x ^ {*} (v; \lambda_ {+} (v)), \bar {y} ^ {*} (v), v; \lambda_ {+} (v)\right) \\ \geq \frac {p - L}{2} \| x ^ {*} (v; \lambda_ {+} (v)) - \bar {x} ^ {*} (v) \| ^ {2}, \tag {82a} \\ \end{array}
$$
$$
\begin{array}{l} K \left(x ^ {*} (v; \lambda_ {+} (v)), \bar {y} ^ {*} (v), v; \lambda_ {+} (v)\right) - K \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v)), v; \lambda_ {+} (v)\right) \\ \geq \frac {p - L}{2} \| y ^ {*} (v; \lambda_ {+} (v)) - \bar {y} ^ {*} (v) \| ^ {2}, \tag {82b} \\ \end{array}
$$
$$
\begin{array}{l} K \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v)), v; \lambda (v)\right) - K \left(x ^ {*} (v; \lambda_ {+} (v)), \bar {y} ^ {*} (v), v; \lambda (v)\right) \\ \geq \frac {p - L}{2} \| y ^ {*} (v; \lambda_ {+} (v)) - \bar {y} ^ {*} (v) \| ^ {2}, \tag {82c} \\ \end{array}
$$
$$
\begin{array}{l} K \left(x ^ {*} (v; \lambda_ {+} (v)), \bar {y} ^ {*} (v), v; \lambda (v)\right) - K (\bar {x} ^ {*} (v), \bar {y} ^ {*} (v), v; \lambda (v)) \\ \geq \frac {p - L}{2} \| x ^ {*} (v; \lambda_ {+} (v)) - \bar {x} ^ {*} (v) \| ^ {2}. \tag {82d} \\ \end{array}
$$
Note that  $\lambda_{+}(v)$  is the maximizer of the following problem.
$$
\max  _ {0 \leq \bar {\lambda} \leq \Lambda} \tau K \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v)), v; \bar {\lambda}\right) - \delta^ {T} (v; \lambda , \lambda_ {+} (v)) \bar {\lambda} \tag {83}
$$
$$
\begin{array}{l} \delta (v; \lambda , \lambda_ {+} (v)) = (\lambda_ {+} (v) + \tau \nabla_ {\lambda} K (x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v)), v; \lambda_ {+} (v))) \\ - \left(\lambda + \tau \nabla_ {\lambda} K \left(x ^ {*} (v; \lambda), y ^ {*} (v; \lambda), v; \lambda\right)\right). \tag {84} \\ \end{array}
$$
According to the Lipschitz continuity of  $\nabla_{\lambda}K$  , we have
$$
\begin{array}{l} \| \delta (v; \lambda , \lambda_ {+} (v)) \| \\ \leq \left\| \lambda_ {+} (v) - \lambda \right\| + \tau \left\| g \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v))\right) - g \left(x ^ {*} (v; \lambda), y ^ {*} (v; \lambda)\right) \right\| \\ + \tau \left\| g ^ {*} \left(x ^ {*} (v; \lambda_ {+} (v))\right) - g ^ {*} \left(x ^ {*} (v; \lambda)\right) \right\| \tag {85} \\ \end{array}
$$
$$
\stackrel {(a)} {\leq} \left(1 + \tau \left(2 \ell_ {g} + \ell_ {g} ^ {\prime}\right) \sigma_ {2}\right) \| \lambda - \lambda_ {+} (v) \| \tag {86}
$$
where in (a) we use the primal error bounds (28a), (28b), and apply the Lipschitz continuity of  $g(\cdot)$  and  $g^{*}(\cdot)$ .
Based on the definition of  $\lambda_{+}(v)$  (cf. (81a)), we have
$$
\begin{array}{l} \tau K \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v)), v; \lambda (v)\right) - \delta^ {T} (v; \lambda , \lambda_ {+} (v)) \lambda (v) \\ \leq \tau K \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v)), v; \lambda_ {+} (v)\right) - \delta^ {T} (v; \lambda , \lambda_ {+} (v)) \lambda_ {+} (v). \tag {87} \\ \end{array}
$$
Subsequently, we can obtain
$$
\begin{array}{l} \tau K \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v)), v; \lambda (v)\right) - \tau K \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v)), v; \lambda_ {+} (v)\right) \\ \leq (\lambda (v) - \lambda_ {+} (v)) ^ {T} \delta (v; \lambda , \lambda_ {+} (v)) \tag {88} \\ \end{array}
$$
$$
\stackrel {(8 6)} {\leq} \| \lambda (v) - \lambda_ {+} (v) \| \left(1 + \tau \left(2 \ell_ {g} + \ell_ {g} ^ {\prime}\right) \sigma_ {2}\right) \| \lambda - \lambda_ {+} (v) \|. \tag {89}
$$
According to the definition of  $\lambda (v)$  (cf. (81b)), we have
$$
K \left(\bar {x} ^ {*} (v), \bar {y} ^ {*} (v), v; \lambda (v)\right) \geq K \left(\bar {x} ^ {*} (v), \bar {y} ^ {*} (v), v; \lambda_ {+} (v)\right). \tag {90}
$$
Combing (82a) to (82d), and (90) yields
$$
\begin{array}{l} 2 \tau (p - L) \left(\| y ^ {*} (v; \lambda_ {+} (v)) - \bar {y} ^ {*} (v) \| ^ {2} + \| x ^ {*} (v; \lambda_ {+} (v)) - \bar {x} ^ {*} (v) \| ^ {2}\right) \\ \leq \left\| \lambda (v) - \lambda_ {+} (v) \right\| \left(1 + \tau \left(2 \ell_ {g} + \ell_ {g} ^ {\prime}\right) \sigma_ {2}\right) \| \lambda - \lambda_ {+} (v) \|. \tag {91} \\ \end{array}
$$
$$
\begin{array}{l} \left\| y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda_ {+} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right) \right\| ^ {2} + \left\| x ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda_ {+} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r}\right) \right\| ^ {2} \\ \leq \underbrace {\frac {1 + \tau \left(2 \ell_ {g} + \ell_ {g} ^ {\prime}\right) \sigma_ {2}}{2 \tau (p - L)}} _ {\triangleq \sigma_ {\text {w e a k}}} \| \lambda^ {r + 1} - \lambda_ {+} \left(w ^ {r}, z ^ {r}\right) \| \| \lambda \left(w ^ {r}, z ^ {r}\right) - \lambda_ {+} \left(w ^ {r}, z ^ {r}\right) \|. \tag {92} \\ \end{array}
$$
B.3.2 Proof of Auxiliary Dual Error Bound
Lemma 8.
Assume that A1-A3, A5 are satisfied and that the local regularity condition holds. Suppose that  $\lambda, y^r, y^*(v; \lambda)$  are bounded, then the following inequality holds.
$$
\begin{array}{l} \| \lambda (w, z) - \lambda_ {+} (w, z) \| \\ \leq \sigma_ {a u x} \left(\| y ^ {*} (w, z; \lambda_ {+} (w, z)) - \bar {y} ^ {*} (w, z) \| + \| x ^ {*} (w, z; \lambda_ {+} (w, z)) - \bar {x} ^ {*} (w, z) \|\right) \tag {93} \\ \end{array}
$$
$$
\sigma_ {a u x} \triangleq \frac {L _ {f} + p + \Lambda L _ {g}}{\sqrt {2 \mu_ {g} \underline {{\delta}}}}. \tag {94}
$$
Proof. From the optimality condition of the block- $y$  update and by following the notations defined in the proof of Lemma 2, we have
$$
\nabla_ {y} f \left(\bar {x} ^ {*} (v), \bar {y} ^ {*} (v)\right) + \lambda (v) \nabla_ {y} g \left(\bar {x} ^ {*} (v), \bar {y} ^ {*} (v)\right) + p \left(\bar {y} ^ {*} (v) - z\right) = 0, \tag {95a}
$$
$$
\begin{array}{l} \nabla_ {y} f \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v))\right) \\ + \lambda_ {+} (v) \nabla_ {y} g \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v)))\right) + p \left(y ^ {*} (v; \lambda_ {+} (v)) - z\right) = 0. \tag {95b} \\ \end{array}
$$
Taking difference of these two equations and using the Lipschitz continuity  $\nabla_y g(\cdot)$ , we have
$$
\begin{array}{l} \left\| \left(\lambda (v) - \lambda_ {+} (v)\right) \nabla_ {y} g \left(\bar {x} ^ {*} (v), \bar {y} ^ {*} (v)\right) \right\| \\ \leq \left\| \nabla_ {y} f (\bar {x} ^ {*} (v), \bar {y} ^ {*} (v)) - \nabla_ {y} f \left(x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v))\right) \right\| \\ + \lambda_ {+} (v) \| \nabla_ {y} g (\bar {x} ^ {*} (v), \bar {y} ^ {*} (v)) - \nabla_ {y} g (x ^ {*} (v; \lambda_ {+} (v)), y ^ {*} (v; \lambda_ {+} (v))) \| + p \| \bar {y} ^ {*} (v) - y ^ {*} (v; \lambda_ {+} (v)) \| \\ \leq \left(L _ {f} + p + \Lambda L _ {g}\right) \left(\| \bar {y} ^ {*} (v) - y ^ {*} (v; \lambda_ {+} (v)) \| + \| \bar {x} ^ {*} (v) - x ^ {*} (v; \lambda_ {+} (v)) \|\right). \tag {96} \\ \end{array}
$$
Further, we know that the left hand side (LHS) of (96) has a lower bound, i.e.,
$$
\left\| \left(\lambda (v) - \lambda_ {+} (v)\right) \nabla_ {\mathcal {Y}} g \left(\bar {x} ^ {*} (v), \bar {y} ^ {*} (v)\right) \right\|
$$
$$
\stackrel {(a)} {\geq} | \lambda (v) - \lambda_ {+} (v) | \sqrt {2 \mu_ {g} \left(g \left(\bar {x} ^ {*} (v) , \bar {y} ^ {*} (v)\right) - g ^ {*} \left(x ^ {*} (v)\right)\right)} \tag {97}
$$
$$
\geq | \lambda (v) - \lambda_ {+} (v) | \sqrt {2 \mu_ {g} \underline {{\delta}}} \tag {98}
$$
where in  $(a)$  we apply the PL condition, and  $(b)$  results from the regularity condition.
B.3.3 Proof of Strong Dual Error Bound (Lemma 3)
Lemma 3 (Formal)
Assume that A1-A5 are satisfied and that the local regularity condition holds. Suppose that  $\lambda, \bar{y}^{*}(w,z), \forall r$  are bounded and  $p > L$ , then the strong error bound holds, namely, there exists a constant  $\sigma_{\text{strong}}$  such that
$$
\begin{array}{l} \left\| y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right) \right\| + \left\| x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r}\right) \right\| \\ \leq 2 \sigma_ {\text {s t r o n g}} \tau \left| g \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right), y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - g ^ {*} \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - \delta \right| \tag {99} \\ \end{array}
$$
$$
\sigma_ {\text {s t r o n g}} \triangleq \sigma_ {\text {w e a k}} \sigma_ {\text {a u x}} = \sigma_ {\text {w e a k}} \frac {L _ {f} + p + \Lambda L _ {g}}{\sqrt {2 \mu_ {g} \underline {{\delta}}}}. \tag {100}
$$
Proof. Under the assumptions that  $p > L$ , and  $\lambda^r, y^r, \bar{y}^*(w^r, z^r)$  are bounded, combing (14) and (93) yields the following strong error bound.
$$
\begin{array}{l} \frac {1}{2} \left(\| y ^ {*} (w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} (w ^ {r}, z ^ {r})) - \bar {y} ^ {*} (w ^ {r}, z ^ {r}) \| + \| x ^ {*} (w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} (w ^ {r}, z ^ {r})) - \bar {x} ^ {*} (w ^ {r}, z ^ {r}) \|\right) ^ {2} \\ \leq \underbrace{\sigma_{\text{weak}}\sigma_{\text{aux}}}_{\triangleq \sigma_{\text{strong}}}\| \lambda^{r + 1} - \lambda_{+}(w^{r},z^{r})\| \\ \cdot \left(\| y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right) \| + \| x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r}\right) \|\right), \\ \end{array}
$$
$$
\begin{array}{l} \left\| y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right) \right\| + \left\| x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r}\right) \right\| \\ \leq 2 \sigma_ {\text {s t r o n g}} \| \lambda^ {r + 1} - \lambda_ {+} ^ {r + 1} \left(w ^ {r}, z ^ {r}\right) \| (101) \\ \leq 2 \sigma_ {\text {s t r o n g}} \tau | \nabla_ {\lambda} K \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right), y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right), z ^ {r}; \lambda^ {r + 1}\right) | (102) \\ \leq 2 \sigma_ {\text {s t r o n g}} \tau | g \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right), y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - g ^ {*} \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - \delta |. (103) \\ \end{array}
$$
B.4 Proof of Potential function (Lemma 1)
Lemma 1 (Formal)
Assume that A1-A5 are satisfied. Suppose that the sequence  $\{x^r, y^r, w^r, z^r, u^r, \lambda^r, \forall r\}$  is generated by SLM,  $p > L$ , and  $\lambda^r, y^r, \bar{y}^*(w^r, z^r)$  are bounded. When  $\gamma, \eta, \alpha, T^r$  respectively satisfy (30), (41) and  $T^r \geq -2 \log_{(1-2\gamma \mu_g)}(\sqrt{\tau} r)$ , then, there exists a constant  $\zeta$  such that
$$
\begin{array}{l} \mathcal {Q} \left(x ^ {r + 1}, y ^ {r + 1}, w ^ {r + 1}, z ^ {r + 1}; \lambda^ {r + 1}\right) - \mathcal {Q} \left(x ^ {r}, y ^ {r}, w ^ {r}, z ^ {r}; \lambda^ {r}\right) \\ \leq - \left(\frac {1}{4 \eta} - \frac {1 7 \tau \left(\ell_ {g} ^ {2} + \ell_ {g} ^ {\prime 2}\right) \sigma_ {3} ^ {2}}{\eta^ {2}}\right) \| x ^ {r + 1} - x ^ {r} \| ^ {2} - \left(\frac {1}{2 \alpha} - \frac {9 \tau \ell_ {g} ^ {2} \sigma_ {3} ^ {2}}{\alpha^ {2}}\right) \| y ^ {r + 1} - y ^ {r} \| ^ {2} \\ \left. - p \left(\frac {1}{2 \beta} - \left(\frac {1}{\zeta} + \frac {2 p}{p - L} + 3 6 \zeta \sigma_ {1} ^ {2}\right)\right) \| z ^ {r + 1} - z ^ {r} \| ^ {2} - p \left(\frac {1}{2 \beta} - \left(\frac {1}{\zeta} + \frac {2 p}{p - L}\right)\right) \| w ^ {r + 1} - w ^ {r} \| ^ {2} \right. \\ + 3 6 p \zeta \| x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r}\right) \| ^ {2} \\ + 3 6 p \zeta \| y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda_ {+} ^ {r + 1} \left(w ^ {r}, z ^ {r}\right)\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right) \| ^ {2} \\ - \tau \left(\frac {1}{5 6} - 4 8 p \zeta \sigma_ {2} ^ {2} \tau\right) | g (x ^ {*} (w ^ {r}, z ^ {r}; \lambda^ {r + 1}), y ^ {*} (w ^ {r}, z ^ {r}; \lambda^ {r + 1})) - g ^ {*} (x ^ {*} (w ^ {r}, z ^ {r}; \lambda^ {r + 1})) - \delta | ^ {2} \\ + \left(\mu_ {g} L _ {g} ^ {2} + 2 \ell_ {g} ^ {2}\right) \frac {D _ {\mathcal {S}}}{r ^ {2}} - \frac {1}{4 \tau} \| \lambda^ {r + 1} - \lambda^ {r} \| ^ {2} \tag {104} \\ \end{array}
$$
$$
\mathcal {Q} ^ {r} \triangleq \mathcal {Q} \left(x ^ {r}, y ^ {r}, w ^ {r}, z ^ {r}; \lambda^ {r}\right) \triangleq K \left(x ^ {r}, y ^ {r}, w ^ {r}, z ^ {r}; \lambda^ {r}\right) - 2 D \left(w ^ {r}, z ^ {r}; \lambda^ {r}\right) + 2 P \left(w ^ {r}, z ^ {r}\right). \tag {105}
$$
$$
\mathcal {Q} ^ {r} = K \left(x ^ {r}, y ^ {r}, w ^ {r}, z ^ {r}; \lambda^ {r}\right) - 2 D \left(w ^ {r}, z ^ {r}; \lambda^ {r}\right) + 2 P \left(w ^ {r}, z ^ {r}\right). \tag {106}
$$
Merging (42), (61), and (74) gives
$$
\begin{array}{l} \mathcal {Q} ^ {r + 1} - \mathcal {Q} ^ {r} \\ \leq - \left(\frac {1}{4 \eta} \| x ^ {r + 1} - x ^ {r} \| ^ {2} + \frac {1}{2 \alpha} \| y ^ {r + 1} - y ^ {r} \| ^ {2} + \frac {p}{2 \beta} \| z ^ {r + 1} - z ^ {r} \| ^ {2} + \frac {p}{2 \beta} \| w ^ {r + 1} - w ^ {r} \| ^ {2}\right) \\ + \left\langle g \left(x ^ {r}, y ^ {r}\right) - g ^ {*} \left(x ^ {r}\right) - \delta , \lambda^ {r + 1} - \lambda^ {r} \right\rangle \\ - 2 \left\langle \lambda^ {r + 1} - \lambda^ {r}, g \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right), y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - g ^ {*} \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - \delta \right\rangle \\ \underbrace{-p\langle z^{r + 1} - z^{r},z^{r + 1} + z^{r} - 2y^{*}(w^{r},z^{r + 1};\lambda^{r + 1})\rangle - 2p(z^{r + 1} - z^{r})^{T}(\bar{y}^{*}(w^{r},z^{r}) - z^{r})}_{\triangleq T_{1}} \\ \underbrace{-p\langle w^{r + 1} - w^{r},w^{r + 1} + w^{r} - 2x^{*}(w^{r + 1},z^{r + 1};\lambda^{r + 1})\rangle - 2p(w^{r + 1} - w^{r})^{T}(\bar{x}^{*}(w^{r},z^{r + 1}) - w^{r})}_{\triangleq T_{2}} \\ + p \left(\frac {p}{p - L} + 1\right) \left(\| z ^ {r + 1} - z ^ {r} \| ^ {2} + \| w ^ {r + 1} - w ^ {r} \| ^ {2}\right) + \frac {\mu_ {g} L _ {g} ^ {2} D _ {\mathcal {S}}}{r ^ {2}}. \tag {107} \\ \end{array}
$$
First, we can get the upper bound of term  $T_{1}$  as follows:
$$
\begin{array}{l} T _ {1} = - p \left\langle z ^ {r + 1} - z ^ {r}, z ^ {r + 1} - z ^ {r} - 2 \left(y ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right)\right) \right\rangle \\ = - p \left\langle z ^ {r + 1} - z ^ {r}, z ^ {r + 1} - z ^ {r} - 2 \left(y ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) - y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) \right\rangle \\ - p \left\langle z ^ {r + 1} - z ^ {r}, y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right) \right\rangle (108) \\ = - p \| z ^ {r + 1} - z ^ {r} \| ^ {2} + 2 p \left\langle z ^ {r + 1} - z ^ {r}, y ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) - y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right) \right\rangle \\ - p \left\langle z ^ {r + 1} - z ^ {r}, y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right) \right\rangle . (109) \\ \end{array}
$$
For the last term of the above inequality, we can further have
$$
\begin{array}{l} p \left\langle z ^ {r + 1} - z ^ {r}, y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right) \right\rangle \\ \leq \frac {p \| z ^ {r + 1} - z ^ {r} \| ^ {2}}{2 \zeta} + \frac {p \zeta}{2} \| y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right) - \bar {y} ^ {*} \left(w ^ {r}, z ^ {r}\right) \| ^ {2}. \tag {110} \\ \end{array}
$$
For the second term of (109), we can get
$$
\left\langle z ^ {r + 1} - z ^ {r}, y ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) - y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right) \right\rangle
$$
$$
\stackrel {(a)} {\leq} \| z ^ {r + 1} - z ^ {r} \| \| y ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) - y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right) \| \tag {111}
$$
$$
\stackrel {(b)} {\leq} \frac {p}{p - L} \| z ^ {r + 1} - z ^ {r} \| ^ {2} \tag {112}
$$
where  $(a)$  is true by applying the Cauchy-Schwarz inequality, in (b) we use the primal error bound (27a).
$$
\begin{array}{l} T _ {2} = - p \left\langle w ^ {r + 1} - w ^ {r}, w ^ {r + 1} - w ^ {r} - 2 \left(x ^ {*} \left(w ^ {r + 1}, z ^ {r + 1}; \lambda^ {r + 1}\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r + 1}\right)\right) \right\rangle \\ = - p \left\langle w ^ {r + 1} - w ^ {r}, w ^ {r + 1} - w ^ {r} - 2 \left(x ^ {*} \left(w ^ {r + 1}, z ^ {r + 1}; \lambda^ {r + 1}\right) - x ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right)\right) \right\rangle \\ - p \left\langle w ^ {r + 1} - w ^ {r}, x ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r + 1}\right) \right\rangle (113) \\ = - p \left\| w ^ {r + 1} - w ^ {r} \right\| ^ {2} + 2 p \left\langle w ^ {r + 1} - w ^ {r}, x ^ {*} \left(w ^ {r + 1}, z ^ {r + 1}; \lambda^ {r + 1}\right) - x ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) \right\rangle \\ - p \left\langle w ^ {r + 1} - w ^ {r}, x ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r + 1}\right) \right\rangle . (114) \\ \end{array}
$$
$$
\begin{array}{l} p \left\langle w ^ {r + 1} - w ^ {r}, x ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r + 1}\right) \right\rangle \\ \leq \frac {p \| w ^ {r + 1} - w ^ {r} \| ^ {2}}{2 \zeta} + \frac {p \zeta}{2} \| x ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) - \bar {x} ^ {*} \left(w ^ {r}, z ^ {r + 1}\right) \| ^ {2}. \tag {115} \\ \end{array}
$$
$$
\left\langle w ^ {r + 1} - w ^ {r}, x ^ {*} \left(w ^ {r + 1}, z ^ {r + 1}; \lambda^ {r + 1}\right) - y ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) \right\rangle
$$
$$
\stackrel {(a)} {\leq} \left\| w ^ {r + 1} - w ^ {r} \right\| \left\| x ^ {*} \left(w ^ {r + 1}, z ^ {r + 1}; \lambda^ {r + 1}\right) - x ^ {*} \left(w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}\right) \right\| \tag {116}
$$
$$
\stackrel {(b)} {\leq} \frac {p}{p - L} \| w ^ {r + 1} - w ^ {r} \| ^ {2} \tag {117}
$$
where  $(a)$  is true by applying the Cauchy-Schwarz inequality, in (b) we use the primal error bound (27b).
Substituting (110), (112), (115), (117) into (107) yields
$$
\begin{array}{l} \mathcal {Q} ^ {r + 1} - \mathcal {Q} ^ {r} \\ \leq - \left(\frac {1}{4 \eta} \| x ^ {r + 1} - x ^ {r} \| ^ {2} + \frac {1}{2 \alpha} \| y ^ {r + 1} - y ^ {r} \| ^ {2}\right) \\ - p \left(\frac {1}{2 \beta} - \left(\frac {1}{\zeta} + \frac {2 p}{p - L}\right)\right) \| z ^ {r + 1} - z ^ {r} \| ^ {2} \\ - p \left(\frac {1}{2 \beta} - \left(\frac {1}{\zeta} + \frac {2 p}{p - L}\right)\right) \| w ^ {r + 1} - w ^ {r} \| ^ {2} \\ + p \zeta \left(\| x ^ {*} (w ^ {r}, z ^ {r + 1}; \lambda^ {r + 1}) - \bar {x} ^ {*} (w ^ {r}, z ^ {r + 1}) \| ^ {2} + \| y ^ {*} (w ^ {r}, z ^ {r}; \lambda^ {r + 1}) - \bar {y} ^ {*} (w ^ {r}, z ^ {r}) \| ^ {2}\right) + \frac {\mu_ {g} L _ {g} ^ {2} D _ {S}}{r ^ {2}} \\ + \langle g (x ^ {r}, y ^ {r}) - g ^ {*} (x ^ {r}) - \delta , \lambda^ {r + 1} - \lambda^ {r} \rangle \\ - 2 \left\langle \lambda^ {r + 1} - \lambda^ {r}, g \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right), y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - g ^ {*} \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - \delta\right) \rangle . \tag {118} \\ \end{array}
$$
Second, we will give an upper bound of the last two terms of (118) as follows.
Step 1.) From the update of the dual variable, we can have
$$
\langle g \left(x ^ {r}, y ^ {r}\right) - g \left(x ^ {r}, u ^ {r + 1}\right) - \delta , \lambda^ {r + 1} - \lambda^ {r} \rangle \geq \frac {1}{\tau} \| \lambda^ {r + 1} - \lambda^ {r} \| ^ {2}, \tag {119}
$$
$$
- \left\langle g \left(x ^ {r}, y ^ {r}\right) - g \left(x ^ {r}, u ^ {r + 1}\right) - \delta , \lambda^ {r + 1} - \lambda^ {r} \right\rangle \leq - \frac {1}{\tau} \| \lambda^ {r + 1} - \lambda^ {r} \| ^ {2}. \tag {120}
$$
$$
\begin{array}{l} \langle g (x ^ {r}, y ^ {r}) - g ^ {*} (x ^ {r}) - \delta , \lambda^ {r + 1} - \lambda^ {r} \rangle \\ = \langle g \left(x ^ {r}, y ^ {r}\right) - g ^ {*} \left(x ^ {r}\right) - \delta , \lambda^ {r + 1} - \lambda^ {r} \rangle + \langle g \left(x ^ {r}, y ^ {r}\right) - g \left(x ^ {r}, u ^ {r + 1}\right) - \delta , \lambda^ {r + 1} - \lambda^ {r} \rangle \\ - \left\langle g \left(x ^ {r}, y ^ {r}\right) - g \left(x ^ {r}, u ^ {r + 1}\right) - \delta , \lambda^ {r + 1} - \lambda^ {r} \right\rangle (121) \\ \leq 2 \langle g (x ^ {r}, y ^ {r}) - g ^ {*} (x ^ {r}) - \delta , \lambda^ {r + 1} - \lambda^ {r} \rangle + \langle g ^ {*} (x ^ {r}) - g (x ^ {r}, u ^ {r + 1}), \lambda^ {r + 1} - \lambda^ {r} \rangle \\ - \frac {1}{\tau} \| \lambda^ {r + 1} - \lambda^ {r} \| ^ {2}. (122) \\ \end{array}
$$
Subsequently, we can derive an upper bound for the sum of the last two terms in (118) as follows.
$$
\begin{array}{l} 2 \left\langle g \left(x ^ {r}, y ^ {r}\right) - g ^ {*} \left(x ^ {r}\right), \lambda^ {r + 1} - \lambda^ {r} \right\rangle \\ - 2 \langle g \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right), y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - g ^ {*} \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right), \lambda^ {r + 1} - \lambda^ {r} \rangle \\ + \left\langle \lambda^ {r + 1} - \lambda^ {r}, g ^ {*} \left(x ^ {r}\right) - g \left(x ^ {r}, u ^ {r + 1}\right) \right\rangle - \frac {1}{\tau} \| \lambda^ {r + 1} - \lambda^ {r} \| ^ {2} \tag {123} \\ \end{array}
$$
$$
\stackrel {(a)} {\leq} 4 \tau \left(\| g (x ^ {r}, y ^ {r}) - g ^ {*} (x ^ {r}) - \left(g \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right), y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right) - g ^ {*} \left(x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right)\right)\right) \| ^ {2}\right)
$$
$$
\begin{array}{l} + \tau \| g ^ {*} \left(x ^ {r}\right) - g \left(x ^ {r}, u ^ {r + 1}\right) \| ^ {2} - \frac {1}{2 \tau} \| \lambda^ {r + 1} - \lambda^ {r} \| ^ {2} (124) \\ \leq 8 \tau \left(2 \left(\ell_ {g} ^ {2} + \ell_ {g} ^ {r 2}\right) \| x ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right) - x ^ {r} \| ^ {2} + \ell_ {g} ^ {2} \| y ^ {*} \left(w ^ {r}, z ^ {r}; \lambda^ {r + 1}\right) - y ^ {r} \| ^ {2}\right) \\ + \tau \| g ^ {*} \left(x ^ {r}\right) - g \left(x ^ {r}, u ^ {r + 1}\right) \| ^ {2} - \frac {1}{2 \tau} \| \lambda^ {r + 1} - \lambda^ {r} \| ^ {2} (125) \\ \end{array}
$$
$$
\begin{array}{l} \stackrel {(b)} {\leq} 8 \tau \sigma_ {3} ^ {2} \left(\frac {2 \left(\ell_ {g} ^ {2} + \ell_ {g} ^ {\prime 2}\right)}{\eta^ {2}} \| x ^ {r + 1} - x ^ {r} \| ^ {2} + \frac {\ell_ {g} ^ {2}}{\alpha^ {2}} \| y ^ {r + 1} - y ^ {r} \| ^ {2}\right) \\ + \tau \| g ^ {*} \left(x ^ {r}\right) - g \left(x ^ {r}, u ^ {r + 1}\right) \| ^ {2} - \frac {1}{2 \tau} \| \lambda^ {r + 1} - \lambda^ {r} \| ^ {2} \tag {126} \\ \end{array}
$$
where in (a) we use the Cauchy-Schwarz inequality, in (b) we apply the primal error bounds (29a) and (29b).
Step 2.) Applying the reverse triangle inequality, we can get