$$
\begin{array}{l} \mathcal {L} _ {\text {r i d g e}} \left(\boldsymbol {\beta} ^ {t} + \alpha \boldsymbol {e} _ {j}\right) \quad \# e _ {j} \text {d e n o t e s a v e c o r t w i t h j - t h e n t r y t o b e 1 a n d 0 o t h e r w i s e} \\ = (\boldsymbol {y} - \boldsymbol {X} (\boldsymbol {\beta} ^ {t} + \alpha \boldsymbol {e} _ {j})) ^ {T} (\boldsymbol {y} - \boldsymbol {X} (\boldsymbol {\beta} ^ {t} + \alpha \boldsymbol {e} _ {j})) + \lambda_ {2} (\boldsymbol {\beta} ^ {t} + \alpha \boldsymbol {e} _ {j}) ^ {T} (\boldsymbol {\beta} ^ {t} + \alpha \boldsymbol {e} _ {j}) \\ \# \text {a p p l y} \\ = (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t} - \alpha \boldsymbol {X e} _ {j}) ^ {T} (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t} - \alpha \boldsymbol {X e} _ {j}) + \lambda_ {2} (\boldsymbol {\beta} ^ {t} + \alpha \boldsymbol {e} _ {j}) ^ {T} (\boldsymbol {\beta} ^ {t} + \alpha \boldsymbol {e} _ {j}) \\ \# d i s t r i b u t e i n t o t h e p a r e n t h e s e s \\ = (\boldsymbol {y} - \boldsymbol {X} \beta^ {t} - \alpha \boldsymbol {X} _ {: j}) ^ {T} (\boldsymbol {y} - \boldsymbol {X} \beta^ {t} - \alpha \boldsymbol {X} _ {: j}) + \lambda_ {2} (\beta^ {t} + \alpha \boldsymbol {e} _ {j}) ^ {T} (\boldsymbol {\beta} ^ {t} + \alpha \boldsymbol {e} _ {j}) \\ \# X _ {: j} \text {d e n o t e s t h e} j \text {- t h c o l u m n o f} X \\ = (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) ^ {T} (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) - 2 \alpha (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) ^ {T} \boldsymbol {X} _ {: j} + \alpha^ {2} \boldsymbol {X} _ {: j} ^ {T} \boldsymbol {X} _ {: j} + \lambda_ {2} \left[ (\boldsymbol {\beta} ^ {t}) ^ {T} \boldsymbol {\beta} ^ {t} + 2 \alpha (\boldsymbol {\beta} ^ {t}) ^ {T} \boldsymbol {e} _ {j} + \alpha^ {2} \right] \\ \# \text {e x p a n d e a c h q u a d r a t i c t e r m i n t o t h r e e s u m m a t i o n t e r m s} \\ = (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) ^ {T} (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) - 2 \alpha (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) ^ {T} \boldsymbol {X} _ {: j} + \alpha^ {2} \boldsymbol {X} _ {: j} ^ {T} \boldsymbol {X} _ {: j} + \lambda_ {2} \left[ (\boldsymbol {\beta} ^ {t}) ^ {T} \boldsymbol {\beta} ^ {t} + 2 \alpha \beta_ {j} ^ {t} + \alpha^ {2} \right] \\ \# \left(\beta^ {t}\right) ^ {T} e _ {j} = \beta_ {j} ^ {t}; e _ {j} t a k e s t h e j - t h c o m p o n e n t o f \beta^ {t} \\ = (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) ^ {T} (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) - 2 \alpha (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) ^ {T} \boldsymbol {X} _ {: j} + \alpha^ {2} \boldsymbol {X} _ {: j} ^ {T} \boldsymbol {X} _ {: j} + \lambda_ {2} \left[ (\boldsymbol {\beta} ^ {t}) ^ {T} \boldsymbol {\beta} ^ {t} + \alpha^ {2} \right] \\ \# \beta_ {j} ^ {t} = 0 \text {b e c a u s e w e a r e o p t i m i z i n g o n a c o o r d i n a t e w i t h c o e f f i c i e n t e q u a l t o 0} \\ = (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) ^ {T} (\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}) + \lambda_ {2} (\boldsymbol {\beta} ^ {t}) ^ {T} \boldsymbol {\beta} ^ {t} + \left(\| \boldsymbol {X}: _ {j} \| _ {2} ^ {2} + \lambda_ {2}\right) \alpha^ {2} - 2 \left(\left(\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}\right) ^ {T} \boldsymbol {X}: _ {j}\right) \alpha \\ \# c o l e c t r e l e v a n t c o e f f i c i e n t s f o r \alpha^ {2} a n d \alpha \\ = \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t}) + \left(\left\| \boldsymbol {X} _ {: j} \right\| _ {2} ^ {2} + \lambda_ {2}\right) \alpha^ {2} - 2 \left(\left(\boldsymbol {y} - \boldsymbol {X} \boldsymbol {\beta} ^ {t}\right) ^ {T} \boldsymbol {X} _ {: j}\right) \alpha \\ \# \text {s i m p l i f y} \mathcal {L} _ {\text {r i d g e}} (\cdot) \\ = \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t}) + \left(\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}\right) \alpha^ {2} + \nabla_ {j} \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t}) \alpha \\ \# \begin{array}{l} \text {t h e j - t h p a r t i a l d e r i v a t i v e i s} \nabla_ {j} \mathcal {L} _ {\mathrm {r i d g e}} (\boldsymbol {\beta} ^ {t}) = 2 (\boldsymbol {X} \boldsymbol {\beta} ^ {t} - \boldsymbol {y}) ^ {T} \boldsymbol {X} _ {: j} \text {a c c o r d i n g t o} \\ \text {L e m m a B . 7 w i t h} \beta_ {j} ^ {t} = 0 \end{array} \\ = \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t}) + \left(\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}\right) \left(\alpha^ {2} + \frac {\nabla_ {j} \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t})}{\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}} \alpha\right) \\ \# \text {e x t r a c t} \\ = \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t}) + \left(\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}\right) \left(\alpha^ {2} + 2 \frac {1}{2} \frac {\nabla_ {j} \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t})}{\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}} \alpha + \left(\frac {1}{2} \frac {\nabla_ {j} \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t})}{\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}}\right) ^ {2} - \left(\frac {1}{2} \frac {\nabla_ {j} \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t})}{\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}}\right) ^ {2}\right) \\ \# \text {a d d a n d s u b t r a c t t h e s a m e t e r} \\ = \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t}) + \left(\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}\right) \left(\left(\alpha + \frac {1}{2} \frac {\nabla_ {j} \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t})}{\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}}\right) ^ {2} - \left(\frac {1}{2} \frac {\nabla_ {j} \mathcal {L} _ {\text {r i d g e}} (\boldsymbol {\beta} ^ {t})}{\| \boldsymbol {X} _ {: j} \| _ {2} ^ {2} + \lambda_ {2}}\right) ^ {2}\right) \\ \# \text {c o m p l e t e t h e s q u a r e} \\ \end{array}
$$