SlowGuess/NeurIPS_2023_OCR huggingface dataset:
$$
\begin{array}{l} \min  _ {\boldsymbol {\theta} _ {\mathrm {s}}, \boldsymbol {\theta} _ {\mathrm {n s}}} \mathcal {L} _ {A O} (\boldsymbol {\theta} _ {\mathrm {s}}, \boldsymbol {\theta} _ {\mathrm {n s}}) = \min  _ {\alpha_ {0}, \gamma_ {0} \geq 0, \alpha , \gamma} \frac {1}{2} \Big (\frac {1}{\sqrt {n}} \left\| \varepsilon + \boldsymbol {\Lambda} ^ {\top} \boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {\Lambda} ^ {\top} \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \gamma + \sqrt {\| \boldsymbol {\theta} _ {0 , \mathrm {s}} \| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2} + \| \boldsymbol {\gamma} \| _ {\ell_ {2}} ^ {2}} \boldsymbol {g} \right\| _ {\ell_ {2}} \\ + \frac {\boldsymbol {h} _ {\mathrm {s}} ^ {\top} \boldsymbol {\theta} _ {0 , \mathrm {s}}}{\sqrt {n}} + \sum_ {i = 1} ^ {t} \gamma_ {i} \frac {\boldsymbol {h} _ {\mathrm {n s}} ^ {\top} \boldsymbol {u} _ {i , \mathrm {n s}}}{\sqrt {n}} - \gamma_ {0} \xi \left. \frac {\| \boldsymbol {h} _ {\mathrm {n s}} ^ {\perp} \| _ {\ell_ {2}}}{\sqrt {n}}\right) _ {+} ^ {2} \\ + \lambda \| \boldsymbol {\theta} _ {0, \mathrm {s}} \| _ {\ell_ {2}} ^ {2} + \lambda \left(\alpha_ {0} ^ {2} + \| \boldsymbol {\alpha} \| _ {\ell_ {2}} ^ {2}\right) - 2 \lambda \left\langle \boldsymbol {U} _ {\mathrm {s}} ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {s}}, \boldsymbol {\alpha} \right\rangle - 2 \lambda \alpha_ {0} \| \left(\boldsymbol {U} _ {\mathrm {s}} ^ {\perp}\right) ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {s}} \| _ {\ell_ {2}} \\ + \lambda \| \boldsymbol {\theta} _ {0, \mathrm {n s}} \| _ {\ell_ {2}} ^ {2} + \lambda \left(\gamma_ {0} ^ {2} + \| \boldsymbol {\gamma} \| _ {\ell_ {2}} ^ {2}\right) - 2 \lambda \left\langle \boldsymbol {U} _ {\mathrm {n s}} ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {n s}}, \boldsymbol {\gamma} \right\rangle - 2 \lambda \gamma_ {0} \sqrt {1 - \xi^ {2}} \left\| \left(\boldsymbol {U} _ {\mathrm {n s}} ^ {\perp}\right) ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {n s}} \right\| _ {\ell_ {2}}. \tag {B.4} \\ \end{array}
$$
Note that at this stage, the AO problem is reduced to an optimization over  $r + t + 3$  scalar variables  $(\alpha_0, \gamma_0 \geq 0, 0 \leq \xi \leq 1$  and  $\pmb{\alpha} \in \mathbb{R}^r, \pmb{\gamma} \in \mathbb{R}^t$ ).
Convergence of the auxiliary optimization problem. We next continue to derive the point-wise in-probability limit of the AO problem.
First observe that since  $\varepsilon$  and  $g$  are independent with i.i.d  $\mathsf{N}(0,1)$  entries, we have
$$
\varepsilon + \sqrt {\left\| \boldsymbol {\theta} _ {0 , \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2} + \left\| \boldsymbol {\gamma} \right\| _ {\ell_ {2}} ^ {2}} \boldsymbol {g} \stackrel {(d)} {=} \sqrt {\sigma^ {2} + ^ {2} \left(\left\| \boldsymbol {\theta} _ {0 , \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2} + \left\| \boldsymbol {\gamma} \right\| _ {\ell_ {2}} ^ {2}\right)} \tilde {\boldsymbol {g}},
$$
where  $\tilde{\pmb{g}}\in \mathbb{R}^n$  has i.i.d  $\mathsf{N}(0,1)$  entries.
Second, by construction  $\boldsymbol{\Lambda}\boldsymbol{\Lambda}^{\top} = \mathrm{diag}(n_1,\dots,n_k)\in \mathbb{R}^{k\times k}$ , where  $n_{\ell}$  denotes the number of examples from cluster  $\ell$ . Hence,
$$
\begin{array}{l} \frac {1}{n} \left\| \boldsymbol {\Lambda} ^ {\mathsf {T}} \boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {\Lambda} ^ {\mathsf {T}} \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma} \right\| _ {\ell_ {2}} ^ {2} = (\boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma}) ^ {\mathsf {T}} \mathrm {d i a g} (\frac {n _ {1}}{n}, \dots , \frac {n _ {k}}{k}) (\boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma}) \\ \stackrel {(p)} {\rightarrow} \left(\boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma}\right) ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \bigl (\boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma} \bigr) \\ \end{array}
$$
Next, by using concentration of Lipschitz functions of Gaussian vectors, we obtain
$$
\begin{array}{l} \frac {1}{\sqrt {n}} \left\| \boldsymbol {\varepsilon} + \boldsymbol {\Lambda} ^ {\mathsf {T}} \boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {\Lambda} ^ {\mathsf {T}} \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma} + \sqrt {\left\| \boldsymbol {\theta} _ {0 , \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2} + \left\| \boldsymbol {\gamma} \right\| _ {\ell_ {2}} ^ {2}} \boldsymbol {g} \right\| _ {\ell_ {2}} \\ \xrightarrow {p} \sqrt {\left(\boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma}\right) ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \left(\boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma}\right) + \sigma^ {2} + \left(\left\| \boldsymbol {\theta} _ {0 , \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2} + \left\| \boldsymbol {\gamma} \right\| _ {\ell_ {2}} ^ {2}\right)} \\ \end{array}
$$
Also, since  $\| \pmb{\theta}_{0,\mathrm{s}}\|_{\ell_2}$  is bounded and  $\| \pmb{u}_{i,\mathrm{s}}\|_{\ell_2} = 1$ , we get
$$
\frac {\boldsymbol {h} _ {\mathrm {s}} ^ {\top} \boldsymbol {\theta} _ {0 , \mathrm {s}}}{\sqrt {n}}, \frac {\boldsymbol {h} _ {\mathrm {n s}} ^ {\top} \boldsymbol {u} _ {i , \mathrm {n s}}}{\sqrt {n}} \stackrel {(p)} {\rightarrow} 0.
$$
In addition,  $\| h_{\mathrm{ns}}^{\perp}\|_{\ell_2}$  concentrates around  $\sqrt{d - p - t}$  and  $(d - p - t) / n\to \psi_d - \psi_p$  , because  $t\leq k$  remains bounded as  $n$  diverges, and so
$$
\frac {\left\| \boldsymbol {h} _ {\mathrm {n s}} ^ {\perp} \right\| _ {\ell_ {2}}}{\sqrt {n}} \stackrel {(p)} {\rightarrow} \sqrt {\psi_ {d} - \psi_ {p}}.
$$
Using the above limits, the objective in (B.4) converges in-probability to
$$
\begin{array}{l} \mathcal {D} \left(\alpha_ {0}, \gamma_ {0}, \xi , \alpha , \gamma\right) := \\ \frac {1}{2} \left(\sqrt {\left(V _ {\mathrm {s}} \pmb {\Sigma} _ {\mathrm {s}} \pmb {\alpha} + V _ {\mathrm {n s}} \pmb {\Sigma} _ {\mathrm {n s}} \pmb {\gamma}\right) ^ {\top} \mathrm {d i a g} (\pmb {\pi}) \left(V _ {\mathrm {s}} \pmb {\Sigma} _ {\mathrm {s}} \pmb {\alpha} + V _ {\mathrm {n s}} \pmb {\Sigma} _ {\mathrm {n s}} \pmb {\gamma}\right) + \sigma^ {2} + \left(\left\| \pmb {\theta} _ {0 , \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2} + \left\| \pmb {\gamma} \right\| _ {\ell_ {2}} ^ {2}\right)} - \gamma_ {0} \xi \sqrt {\psi_ {d} - \psi_ {p}}\right) _ {+} ^ {2} \\ + \lambda \| \boldsymbol {\theta} _ {0} \| _ {\ell_ {2}} ^ {2} + \lambda \left(\alpha_ {0} ^ {2} + \gamma_ {0} ^ {2} + \| \boldsymbol {\alpha} \| _ {\ell_ {2}} ^ {2} + \| \boldsymbol {\gamma} \| _ {\ell_ {2}} ^ {2}\right) \\ - 2 \lambda \left(\left\langle U _ {\mathrm {s}} ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {s}}, \boldsymbol {\alpha} \right\rangle + \alpha_ {0} \left\| \left(U _ {\mathrm {s}} ^ {\perp}\right) ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} + \left\langle U _ {\mathrm {n s}} ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {n s}}, \boldsymbol {\gamma} \right\rangle + \gamma_ {0} \sqrt {1 - \xi^ {2}} \left\| \left(U _ {\mathrm {n s}} ^ {\perp}\right) ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {n s}} \right\| _ {\ell_ {2}}\right) \tag {B.5} \\ \end{array}
$$
We are now ready to prove the theorems.
B.2.1 Proof of Theorem 3.1
Using Lemma 2.1, we have
$$
\begin{array}{l} \operatorname {R i s k} \left(\widehat {\boldsymbol {\theta}} _ {L}\right) = \sigma^ {2} + \left\| \boldsymbol {\theta} _ {0} - \boldsymbol {\theta} \right\| _ {\ell_ {2}} ^ {2} + \left(\boldsymbol {\theta} _ {0} - \boldsymbol {\theta}\right) ^ {\top} M \operatorname {d i a g} (\boldsymbol {\pi}) M ^ {\top} \left(\boldsymbol {\theta} _ {0} - \boldsymbol {\theta}\right) \\ = \sigma^ {2} + \| q _ {\mathrm {s}} \| _ {\ell_ {2}} ^ {2} + \| q _ {\mathrm {n s}} \| _ {\ell_ {2}} ^ {2} + q _ {\mathrm {s}} ^ {\top} M _ {\mathrm {s}} \operatorname {d i a g} (\pi) M _ {\mathrm {s}} ^ {\top} q _ {\mathrm {s}} + q _ {\mathrm {n s}} ^ {\top} M _ {\mathrm {n s}} \operatorname {d i a g} (\pi) M _ {\mathrm {n s}} ^ {\top} q _ {\mathrm {n s}} \\ = \sigma^ {2} + \left(\alpha_ {0} ^ {2} + \gamma_ {0} ^ {2} + \| \boldsymbol {\alpha} \| _ {\ell_ {2}} ^ {2} + \| \boldsymbol {\gamma} \| _ {\ell_ {2}} ^ {2}\right) \\ + \boldsymbol {\alpha} ^ {\top} \boldsymbol {\Sigma} _ {s} \mathbf {V} _ {s} ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \mathbf {V} _ {s} \boldsymbol {\Sigma} _ {s} \boldsymbol {\alpha} + \boldsymbol {\gamma} ^ {\top} \boldsymbol {\Sigma} _ {n s} \mathbf {V} _ {n s} ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \mathbf {V} _ {n s} \boldsymbol {\Sigma} _ {n s} \boldsymbol {\gamma}. \tag {B.6} \\ \end{array}
$$
Since  $\psi_d - \psi_p \leq 1$ , we are in the over-determined (a.k.a underparametrized) regime. As  $\lambda \to 0^+$ , the terms involving  $\lambda$  become negligible compared to the first term in (B.5) except those that include  $\alpha_0$ , as  $\alpha_0$  is not present in the first term. Since  $(x)_+^2$  is increasing, and
$$
\left(\boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma}\right) ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \left(\boldsymbol {V} _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + \boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma}\right) + \| \boldsymbol {\gamma} \| _ {\ell_ {2}} ^ {2} \geq 0,
$$
the minimum over  $\alpha$  and  $\gamma$  is achieved for  $\alpha = 0 \in \mathbb{R}^r$  and  $\gamma = 0 \in \mathbb{R}^t$ . The optimization (B.5) then reduces to
$$
\min  _ {\alpha_ {0}, \gamma_ {0} \geq 0, 0 \leq \xi \leq 1} \frac {1}{2} \left(\sqrt {\sigma^ {2} + \left(\left\| \boldsymbol {\theta} _ {0 , s} \right\| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2}\right)} - \gamma_ {0} \xi \sqrt {\psi_ {d} - \psi_ {p}}\right) _ {+} ^ {2} + \lambda \alpha_ {0} ^ {2} - 2 \lambda \alpha_ {0} \left\| \left(\boldsymbol {U} _ {\mathrm {s}} ^ {\perp}\right) ^ {\mathsf {T}} \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}}. \tag {B.7}
$$
The optimal  $\xi$  is given by  $\xi = 1$ . Also, setting derivative with respect to  $\alpha_0$  to zero we obtain the optimal  $\alpha_0 = \left\| (\mathbf{U}_{\mathrm{s}}^{\perp})^{\mathsf{T}}\pmb{\theta}_{0,\mathrm{s}}\right\|_{\ell_2}$ . Next, by setting derivative with respect to  $\gamma_0$  we arrive at
$$
\gamma_ {0} ^ {2} = \left(\sigma^ {2} + \left\| \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2}\right) \frac {\psi_ {d} - \psi_ {p}}{1 - \left(\psi_ {d} - \psi_ {p}\right)}.
$$
Using the optimal variables in (B.6) we obtain the risk of minimum-norm estimator as
$$
\begin{array}{l} \operatorname {R i s k} \left(\widehat {\boldsymbol {\theta}} _ {L}\right) = \sigma^ {2} + \left\| \left(\mathbf {U} _ {\mathrm {s}} ^ {\perp}\right) ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2} + \left(\sigma^ {2} + \left\| \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2}\right) \frac {\psi_ {d} - \psi_ {p}}{1 - \left(\psi_ {d} - \psi_ {p}\right)} \\ = \left(\sigma^ {2} + \left\| \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2}\right) \frac {1}{1 - \left(\psi_ {d} - \psi_ {p}\right)} - \left\| \boldsymbol {U} _ {\mathrm {s}} ^ {\mathsf {T}} \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2}. \\ \end{array}
$$
Recall that by assumption,  $r_{\mathrm{s}} = \left\| \pmb{\theta}_{0,\mathrm{s}}\right\|_{\ell_2}$  and  $\left\| \pmb{U}_{\mathrm{s}}^{\mathsf{T}}\pmb{\theta}_{0,\mathrm{s}}\right\|_{\ell_2} = \sqrt{\rho} r_{\mathrm{s}}$  which completes the proof.
B.2.2 Proof of Theorem 3.2
We continue from (B.5). In the case of  $\psi_d - \psi_p \leq 1$ , it is easy to see that the derivative of the first term of (B.5), in the active region is decreasing in  $\gamma_0$ . With the consideration  $\lambda \to 0^+$ , minimizing over  $\gamma_0$  will push us into the non-active region. Therefore the optimization problem (B.5) reduces to
minimize  $\| \pmb{\theta}_0\|_{\ell_2}^2 +\alpha_0^2 +\gamma_0^2 +\| \pmb {\alpha}\|_{\ell_2}^2 +\| \pmb {\gamma}\|_{\ell_2}^2$
$$
- 2 \left(\left\langle \boldsymbol {U} _ {\mathrm {s}} ^ {\mathsf {T}} \boldsymbol {\theta} _ {0, \mathrm {s}}, \boldsymbol {\alpha} \right\rangle + \alpha_ {0} \left\| \left(\boldsymbol {U} _ {\mathrm {s}} ^ {\perp}\right) ^ {\mathsf {T}} \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} + \left\langle \boldsymbol {U} _ {\mathrm {n s}} ^ {\mathsf {T}} \boldsymbol {\theta} _ {0, \mathrm {n s}}, \boldsymbol {\gamma} \right\rangle + \gamma_ {0} \sqrt {1 - \xi^ {2}} \left\| \left(\boldsymbol {U} _ {\mathrm {n s}} ^ {\perp}\right) ^ {\mathsf {T}} \boldsymbol {\theta} _ {0, \mathrm {n s}} \right\| _ {\ell_ {2}}\right)
$$
$$
\left(V _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + V _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma}\right) ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \left(V _ {\mathrm {s}} \boldsymbol {\Sigma} _ {\mathrm {s}} \boldsymbol {\alpha} + V _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \boldsymbol {\gamma}\right) + \sigma^ {2} + \left(\left\| \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2} + \left\| \boldsymbol {\gamma} \right\| _ {\ell_ {2}} ^ {2}\right) \leq \gamma_ {0} ^ {2} \xi^ {2} \left(\psi_ {d} - \psi_ {p}\right) \tag {B.8}
$$
By Assumption 2,  $\pmb{\Sigma}_{\mathrm{s}} = \mu \pmb{I}_k$ $V_{\mathrm{s}} = I_{k\times k}$ , and  $\pmb{\Sigma}_{\mathrm{ns}} = \mathbf{0}$ $U_{\mathrm{ns}} = \mathbf{0}$  (no cluster structure on non-sensitive features and an orthogonal, equal energy cluster centers on the sensitive features). Therefore, by fixing  $\gamma \coloneqq \| \gamma \|_{\ell_2}$ , the optimization problem (B.8) becomes:
minimize  $\alpha_0^2 +\gamma_0^2 +\| \pmb {\alpha}\|_{\ell_2}^2 +\gamma^2 -2\left(\langle \pmb {U}_\mathrm{s}^\top \pmb{\theta}_{0,\mathrm{s}},\pmb {\alpha}\rangle +\alpha_0\left\| (\pmb {U}_\mathrm{s}^\perp)^\top \pmb{\theta}_{0,\mathrm{s}}\right\|_{\ell_2} + \gamma_0\sqrt{1 - \xi^2}\| \pmb{\theta}_{0,\mathrm{ns}}\|_{\ell_2}\right)$
$$
\mu^ {2} \boldsymbol {\alpha} ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \boldsymbol {\alpha} + \sigma^ {2} + \| \boldsymbol {\theta} _ {0, s} \| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2} + \gamma^ {2} \leq \gamma_ {0} ^ {2} \xi^ {2} (\psi_ {d} - \psi_ {p}). \tag {B.9}
$$
Since  $\alpha_0$  does not appear in the constraint, it is easy to see that its optimal value is given by  $\alpha_0 = \left\| \left(\boldsymbol{U}_{\mathrm{s}}^{\perp}\right)^{\mathsf{T}}\boldsymbol{\theta}_{0,\mathrm{s}}\right\|_{\ell_2}$ . Also, note that by decreasing  $\gamma$  the objective value decreases and also by the constraint on the other variables become more relaxed. Consequently, the optimal value of  $\gamma$  is  $\gamma = 0$ . Removing  $\alpha_0$  from the objective function, we are left with
minimize  $\gamma_0^2 +\| \pmb {\alpha}\|_{\ell_2}^2 -2\left(\langle \pmb{U}_{\mathrm{s}}^\top \pmb{\theta}_{0,\mathrm{s}},\pmb {\alpha}\rangle +\gamma_0\sqrt{1 - \xi^2}\| \pmb{\theta}_{0,\mathrm{ns}}\|_{\ell_2}\right)$
subject to  $\mu^2\pmb{\alpha}^\top \mathrm{diag}(\pmb {\pi})\pmb {\alpha} + \sigma^2 +\| \pmb{\theta}_{0,\mathrm{s}}\|_{\ell_2}^2 +\gamma_0^2\leq \gamma_0^2\xi^2\big(\psi_d - \psi_p\big).$  (B.10)
Optimal choice of  $\xi$  results in the constraint to become equality. Solving for  $\xi$ , the optimization reduces to
minimize  $\gamma_0^2 +\| \pmb {\alpha}\|_{\ell_2}^2 -2\left(\langle \pmb{U}_{\mathrm{s}}^\top \pmb{\theta}_{0,\mathrm{s}},\pmb {\alpha}\rangle +\sqrt{\gamma_0^2 - \frac{\mu^2\pmb{\alpha}^\top\mathrm{diag}(\pmb{\pi})\pmb{\alpha} + \sigma^2 + \|\pmb{\theta}_{0,\mathrm{s}}\|_{\ell_2}^2 + \gamma_0^2}{\psi_d - \psi_p}}\|\pmb{\theta}_{0,\mathrm{ns}}\|_{\ell_2}\right)$
Setting derivative with respect to  $\gamma_0$  to zero, we obtain
$$
\sqrt {\gamma_ {0} ^ {2} - \frac {\mu^ {2} \boldsymbol {\alpha} ^ {\mathrm {T}} \operatorname {d i a g} (\boldsymbol {\pi}) \boldsymbol {\alpha} + \sigma^ {2} + \left\| \boldsymbol {\theta} _ {0 , \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2}}{\psi_ {d} - \psi_ {p}}} = \left(1 - \frac {1}{\psi_ {d} - \psi_ {p}}\right) \left\| \boldsymbol {\theta} _ {0, \mathrm {n s}} \right\| _ {\ell_ {2}}. \tag {B.11}
$$
Setting derivative with respect to  $\alpha$  to zero and using the previous stationary equation, we get
$$
\boldsymbol {\alpha} = \left(\boldsymbol {I} + \frac {\mu^ {2} \operatorname {d i a g} (\pi)}{\psi_ {d} - \psi_ {p} - 1}\right) ^ {- 1} \boldsymbol {U} _ {\mathrm {s}} ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {s}}. \tag {B.12}
$$
We next square both sides of (B.12) and rearrange the terms to get
$$
\begin{array}{l} \gamma_ {0} ^ {2} = \frac {1}{\psi_ {d} - \psi_ {p} - 1} \left(\sigma^ {2} + \| \boldsymbol {\theta} _ {0, \mathrm {s}} \| _ {\ell_ {2}} ^ {2} + \mu^ {2} \boldsymbol {\alpha} ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \boldsymbol {\alpha}\right) + \left(1 - \frac {1}{\psi_ {d} - \psi_ {p}}\right) \| \boldsymbol {\theta} _ {0, \mathrm {n s}} \| _ {\ell_ {2}} ^ {2} \\ = \frac {1}{\psi_ {d} - \psi_ {p} - 1} \left(\sigma^ {2} + r _ {\mathrm {s}} ^ {2} + \mu^ {2} \boldsymbol {\alpha} ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \boldsymbol {\alpha}\right) + \left(1 - \frac {1}{\psi_ {d} - \psi_ {p}}\right) r _ {\mathrm {n s}} ^ {2}, \\ \end{array}
$$
which are the same expressions for  $\alpha$  and  $\gamma_0$  given in the theorem statement.
The final step is to write the risk of estimator in terms of  $\alpha$ ,  $\gamma_0$ . Invoke equation (B.6), and recall that in the current case,  $\pmb{\Sigma}_{\mathrm{ns}} = \mathbf{0}$ ,  $\pmb{\Sigma}_{\mathrm{s}} = \mu \pmb{I}$ . Also, as we showed in our derivation,  $\gamma = \| \pmb{\gamma} \|_{\ell_2} = 0$ ,  $\alpha_0 = \left\| (\pmb{U}_{\mathrm{s}}^{\perp})^{\top} \pmb{\theta}_{0,\mathrm{s}} \right\|_{\ell_2}$ , by which we arrive at
$$
\begin{array}{l} \operatorname {R i s k} \left(\widehat {\boldsymbol {\theta}} _ {L}\right) = \mu^ {2} \boldsymbol {\alpha} ^ {\top} \operatorname {d i a g} (\boldsymbol {\pi}) \boldsymbol {\alpha} + \sigma^ {2} + \left(\left\| \left(\boldsymbol {U} _ {\mathrm {s}} ^ {\perp}\right) ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} ^ {2} + \gamma_ {0} ^ {2} + \| \boldsymbol {\alpha} \| _ {\ell_ {2}} ^ {2}\right) \\ = \sigma^ {2} + (1 - \rho) r _ {\mathrm {s}} ^ {2} + \gamma_ {0} ^ {2} + \boldsymbol {\alpha} ^ {\top} (I + \mu^ {2} \operatorname {d i a g} (\boldsymbol {\pi})) \boldsymbol {\alpha}. \tag {B.13} \\ \end{array}
$$
B.3 Proof of Theorem 3.3
We follow the proof strategy used for Theorem 3.1-3.2. Here, we would like to characterize the risk of min-norm estimator  $\widehat{\pmb{\theta}}$ . The features matrix has a clustering structure, but the learner is not using that (no look-alike clustering) and is just compute the min-norm estimator for fitting the responses to individual features. Therefore, one can think of this setting as a special case of our previous analysis when there is no sensitive features (so  $\psi_p = 0$ ).
(a) By setting  $\psi_p = 0$  and  $r_{\mathrm{s}} = 0$  in the result of Theorem 3.1, we get that when  $\psi_d \leq 1$ ,
$$
\operatorname {R i s k} (\widehat {\pmb {\theta}}) = \frac {\sigma^ {2}}{1 - \psi_ {d}}.
$$
(b) In this case, we specialize the proof of Theorem 3.2 to the case that  $\psi_p = 0$ . Continuing from (B.8), and removing the terms corresponding to sensitive features, we arrive at
$$
\text {m i n i m i z e} \quad \gamma_ {0} ^ {2} + \| \boldsymbol {\gamma} \| _ {\ell_ {2}} ^ {2} - 2 \left(\left\langle \boldsymbol {U} _ {\mathrm {n s}} ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {n s}}, \boldsymbol {\gamma} \right\rangle + \gamma_ {0} \sqrt {1 - \xi^ {2}} \left\| \left(\boldsymbol {U} _ {\mathrm {n s}} ^ {\perp}\right) ^ {\top} \boldsymbol {\theta} _ {0, \mathrm {n s}} \right\| _ {\ell_ {2}}\right)
$$
$$
\left(\boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \gamma\right) ^ {\top} \operatorname {d i a g} (\pi) \left(\boldsymbol {V} _ {\mathrm {n s}} \boldsymbol {\Sigma} _ {\mathrm {n s}} \gamma\right) + \sigma^ {2} + \gamma_ {0} ^ {2} + \| \gamma \| _ {\ell_ {2}} ^ {2} \leq \gamma_ {0} ^ {2} \xi^ {2} \psi_ {d} \tag {B.14}
$$
We drop the index 'ns' as it is not relevant in this case. Also by Assumption 2,  $\pmb{\Sigma}_{\mathrm{ns}} = \mu \pmb{I}_d$ $V_{\mathrm{ns}} = I_d$ . Therefore, the above optimization can be written as
$$
\text {m i n i m i z e} \quad \gamma_ {0} ^ {2} + \| \boldsymbol {\gamma} \| _ {\ell_ {2}} ^ {2} - 2 \left(\left\langle \boldsymbol {U} ^ {\top} \boldsymbol {\theta} _ {0}, \boldsymbol {\gamma} \right\rangle + \gamma_ {0} \sqrt {1 - \xi^ {2}} \| \left(\boldsymbol {U} ^ {\perp}\right) ^ {\top} \boldsymbol {\theta} _ {0} \| _ {\ell_ {2}}\right)
$$
$$
\text {s u b j e c t} \quad \gamma^ {\top} (I + \mu^ {2} \operatorname {d i a g} (\pi)) \gamma + \sigma^ {2} + \gamma_ {0} ^ {2} \leq \gamma_ {0} ^ {2} \xi^ {2} \psi_ {d}. \tag {B.15}
$$
Optimal  $\xi$  makes the constraint equality. Solving for  $\xi$ , the above optimization can be written as so we have
$$
\text {m i n i m i z e} \quad \gamma_ {0} ^ {2} + \| \boldsymbol {\gamma} \| _ {\ell_ {2}} ^ {2} - 2 \left(\langle \boldsymbol {U} ^ {\mathsf {T}} \boldsymbol {\theta} _ {0}, \boldsymbol {\gamma} \rangle + \sqrt {\gamma_ {0} ^ {2} - \frac {\gamma^ {\mathsf {T}} (\boldsymbol {I} + \mu^ {2} \operatorname {d i a g} (\boldsymbol {\pi})) \boldsymbol {\gamma} + \sigma^ {2} + \gamma_ {0} ^ {2}}{\psi_ {d}}} \| (\boldsymbol {U} ^ {\perp}) ^ {\mathsf {T}} \boldsymbol {\theta} _ {0} \| _ {\ell_ {2}}\right).
$$
Setting the derivative with respect to  $\gamma_0$  to zero, we get
$$
\sqrt {\gamma_ {0} ^ {2} - \frac {\gamma^ {\top} (\boldsymbol {I} + \mu^ {2} \operatorname {d i a g} (\boldsymbol {\pi})) \boldsymbol {\gamma} + \sigma^ {2} + \gamma_ {0} ^ {2}}{\psi_ {d}}} = \left(1 - \frac {1}{\psi_ {d}}\right) \left\| \left(\boldsymbol {U} ^ {\perp}\right) ^ {\top} \boldsymbol {\theta} _ {0} \right\| _ {\ell_ {2}}. \tag {B.16}
$$
Setting derivative with respect to  $\gamma$  to zero and using the above equation, we obtain
$$
\boldsymbol {\gamma} = \left(\boldsymbol {I} + \frac {\boldsymbol {I} + \mu^ {2} \operatorname {d i a g} (\boldsymbol {\pi})}{\psi_ {d} - 1}\right) ^ {- 1} \boldsymbol {U} ^ {\top} \boldsymbol {\theta} _ {0}. \tag {B.17}
$$
We next square both sides of equation (B.16), and rearrange the terms to get:
$$
\gamma_ {0} ^ {2} = \frac {1}{\psi_ {d} - 1} \left(\sigma^ {2} + \gamma^ {\top} (\boldsymbol {I} + \mu^ {2} \operatorname {d i a g} (\boldsymbol {\pi})) \gamma\right) + \left(1 - \frac {1}{\psi_ {d}}\right) \| (\boldsymbol {U} ^ {\perp}) ^ {\top} \boldsymbol {\theta} _ {0} \| _ {\ell_ {2}} ^ {2}.
$$
Under the simplifying Assumption 2, there is no cluster structure on the non-sensitive features and so  $U_{\mathrm{ns}} = 0$ . Therefore,
$$
\left\| \boldsymbol {U} ^ {\mathsf {T}} \boldsymbol {\theta} _ {0} \right\| _ {\ell_ {2}} = \left\| \boldsymbol {U} _ {\mathrm {s}} ^ {\mathsf {T}} \boldsymbol {\theta} _ {0, \mathrm {s}} \right\| _ {\ell_ {2}} = \sqrt {\rho} r _ {\mathrm {s}},
$$