B.5 Derivation of Equation ( 12 )
By using chain rule,

\frac{\partial\operatorname{vec}[g(F)]}{\partial\operatorname{vec}[F]} =\frac{\partial}{\partial\operatorname{vec}[F]}\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}\operatorname{vec}[F^{\top}F] =\frac{\partial}{\partial\operatorname{vec}[F]}\frac{1}{\sqrt{\operatorname{vec}[F^{\top}F]^{\top}\operatorname{vec}[F^{\top}F]}+\epsilon_{F}}\operatorname{vec}[F^{\top}F] =\left(\left.\frac{\partial\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}v}{\partial v}\right|_{v=\operatorname{vec}[F^{\top}F]}\right)\frac{\partial\operatorname{vec}[F^{\top}F]}{\partial\operatorname{vec}[F]}.

For the first term, using chain rule,

\frac{\partial\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}v}{\partial v} =\left(\left.\frac{\partial av}{\partial a}\right|_{a=\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}}\right)\left(\frac{\partial\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}}{\partial v}\right)+\left.\frac{\partial av}{\partial v}\right|_{a=\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}} =v\left(\frac{\partial\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}}{\partial v}\right)+\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}I_{mm} =v\left(\left.\frac{\partial a^{-1}}{\partial a}\right|_{a=\sqrt{v^{\top}v}+\epsilon_{F}}\right)\left(\frac{\partial\sqrt{v^{\top}v}+\epsilon_{F}}{\partial v}\right)+\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}I_{mm} =v\left(-(\sqrt{v^{\top}v}+\epsilon_{F})^{-2}\right)\left(\frac{\partial\sqrt{v^{\top}v}}{\partial v}\right)+\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}I_{mm} =v\left(-(\sqrt{v^{\top}v}+\epsilon_{F})^{-2}\right)\left(\left.\frac{\partial\sqrt{a}}{\partial a}\right|_{a=v^{\top}v}\right)\left(\frac{\partial v^{\top}v}{\partial v}\right)+\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}I_{mm} =v\left(-(\sqrt{v^{\top}v}+\epsilon_{F})^{-2}\right)\left(\frac{1}{2}(v^{\top}v)^{-1/2}\right)\left(2v^{\top}\right)+\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}I_{mm} =-\frac{1}{(\sqrt{v^{\top}v}+\epsilon_{F})^{2}}\frac{1}{\sqrt{v^{\top}v}}vv^{\top}+\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}I_{mm} =\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}\left(I_{mm}-\frac{1}{(\sqrt{v^{\top}v}+\epsilon_{F})\sqrt{v^{\top}v}}vv^{\top}\right)

Thus,

\left.\frac{\partial\frac{1}{\sqrt{v^{\top}v}+\epsilon_{F}}v}{\partial v}\right|_{v=\operatorname{vec}[F^{\top}F]}=\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}\left(I_{mm}-\frac{1}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}\right)

For the second term,

\frac{\partial F^{\top}F}{\partial F_{ij}}=F^{\top}\Delta^{ij}+\Delta^{ji}F,

where \Delta^{ij}\in\mathbb{R}^{m\times m} is the matrix with the (i,j) -th entry being one and all other entries being zero.
Using vectorization and the square commutation matrix K^{(m,m)} ,

\frac{\partial\operatorname{vec}[F^{\top}F]}{\partial F_{ij}} =\operatorname{vec}[F^{\top}\Delta^{ij}]+\operatorname{vec}[\Delta^{ji}F] =\operatorname{vec}[F^{\top}\Delta^{ij}]+\operatorname{vec}[(F^{\top}(\Delta^{ji})^{\top})^{\top}] =\operatorname{vec}[F^{\top}\Delta^{ij}]+\operatorname{vec}[(F^{\top}\Delta^{ij})^{\top}] =\operatorname{vec}[F^{\top}\Delta^{ij}]+K^{(m,m)}\operatorname{vec}[F^{\top}\Delta^{ij}] =(I_{mm}+K^{(m,m)})\operatorname{vec}[F^{\top}\Delta^{ij}] =(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}]\operatorname{vec}[\Delta^{ij}].

Thus,

\frac{\partial\operatorname{vec}[F^{\top}F]}{\partial\operatorname{vec}[F]} =(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}][\operatorname{vec}[\Delta^{11}],\dots,\operatorname{vec}[\Delta^{m1}],\operatorname{vec}[\Delta^{12}],\dots,\operatorname{vec}[\Delta^{m2}],\cdots,\operatorname{vec}[\Delta^{1m}],\dots,\operatorname{vec}[\Delta^{mm}]] =(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}]I_{mm} =(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}].

Combining the first term and second term,

\frac{\partial\operatorname{vec}[g(F)]}{\partial\operatorname{vec}[F]} =\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}\left(I_{mm}-\frac{1}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}\right)(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}].

Since the square commutation matrix
is symmetric, using the definition of the commutation matrix,

\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}K^{(m,m)} =\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}(K^{(m,m)})^{\top} =\operatorname{vec}[F^{\top}F](K^{(m,m)}\operatorname{vec}[F^{\top}F])^{\top} =\operatorname{vec}[F^{\top}F](\operatorname{vec}[(F^{\top}F)^{\top}])^{\top} =\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}

Therefore,

\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}(I_{mm}+K^{(m,m)})=2\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}.

Using this,

\frac{\partial\operatorname{vec}[g(F)]}{\partial\operatorname{vec}[F]} =\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}\left(I_{mm}-\frac{1}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}\right)(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}] =\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}\left(I_{mm}+K^{(m,m)}-\frac{2}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}\right)[I_{m}\otimes F^{\top}] =\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}]-\frac{2}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[F^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}[I_{m}\otimes F^{\top}] =\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}]-\frac{2}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[F^{\top}F]([I_{m}\otimes F]\operatorname{vec}[F^{\top}F])^{\top} =\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}]-\frac{2}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[F^{\top}F]\operatorname{vec}[FF^{\top}F]^{\top}

By using this form of \frac{\partial\operatorname{vec}[g(F)]}{\partial\operatorname{vec}[F]} and by defining R=Q_{F}\left(G\circ\left(Q_{F}^{\top}B^{\top}\frac{\partial\ell_{Y}(\mathbf{f}_{X,F,B})}{\partial\mathbf{f}_{X,F,B}}Q_{S}\right)\right)Q_{S}^{\top}SZ_{X,F}^{\top} ,

 abla_{\operatorname{vec}[F]}L(B,F) =\gamma\left(\frac{\partial\operatorname{vec}[g(F)]}{\partial\operatorname{vec}[F]}\right)^{\top}\operatorname{vec}\left[R\right], =\gamma\left(\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}(I_{mm}+K^{(m,m)})[I_{m}\otimes F^{\top}]-\frac{2}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[F^{\top}F]\operatorname{vec}[FF^{\top}F]^{\top}\right)^{\top}\operatorname{vec}\left[R\right] =\gamma\left(\frac{1}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}[I_{m}\otimes F](I_{mm}+K^{(m,m)})-\frac{2}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[FF^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}\right)\operatorname{vec}\left[R\right] =\left(\frac{\gamma}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}[I_{m}\otimes F](I_{mm}+K^{(m,m)})-\frac{2\gamma}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[FF^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}\right)\operatorname{vec}\left[R\right]

We compute each of cross terms:

[I_{m}\otimes F]I_{mm}\operatorname{vec}\left[R\right]=\operatorname{vec}\left[FR\right]

[I_{m}\otimes F]K^{(m,m)}\operatorname{vec}\left[R\right]=[I_{m}\otimes F]\operatorname{vec}\left[R^{\top}\right]=\operatorname{vec}\left[FR^{\top}\right]

\operatorname{vec}[FF^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}\operatorname{vec}\left[R\right]=\operatorname{vec}[FF^{\top}F]\left\langle F^{\top}F,R\right\rangle_{\mathrm{F}}

Using these,
since vectorization \operatorname{vec} is a linear map,

 abla_{\operatorname{vec}[F]}L(B,F) =\left(\frac{\gamma}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}[I_{m}\otimes F](I_{mm}+K^{(m,m)})-\frac{2\gamma}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[FF^{\top}F]\operatorname{vec}[F^{\top}F]^{\top}\right)\operatorname{vec}\left[R\right] =\frac{\gamma}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}(\operatorname{vec}\left[FR\right]+\operatorname{vec}\left[FR^{\top}\right])-\frac{2\gamma\left\langle F^{\top}F,R\right\rangle_{\mathrm{F}}}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[FF^{\top}F] =\frac{\gamma}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}\operatorname{vec}\left[FR+FR^{\top}\right]-\frac{2\gamma\left\langle F^{\top}F,R\right\rangle_{\mathrm{F}}}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[FF^{\top}F] =\frac{\gamma}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}\operatorname{vec}\left[F(R+R^{\top})\right]-\frac{2\gamma\left\langle F^{\top}F,R\right\rangle_{\mathrm{F}}}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[FF^{\top}F]

Therefore,

abla_{\operatorname{vec}[F]}L(B,F)=\frac{\gamma}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}\operatorname{vec}\left[F(R+R^{\top})\right]-\frac{2\gamma\left\langle F^{\top}F,R\right\rangle_{\mathrm{F}}}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}\operatorname{vec}[FF^{\top}F]

 abla_{F}L(B,F) =\frac{\gamma}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}F(R+R^{\top})-\frac{2\gamma\left\langle F^{\top}F,R\right\rangle_{\mathrm{F}}}{(\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F})^{2}\|F^{\top}F\|_{\mathrm{F}}}FF^{\top}F =\frac{\gamma}{\|F^{\top}F\|_{\mathrm{F}}+\epsilon_{F}}F\left((R+R^{\top})-\frac{2\left\langle F^{\top}F,R\right\rangle_{\mathrm{F}}}{\|F^{\top}F\|_{\mathrm{F}}^{2}+\epsilon_{F}\|F^{\top}F\|_{\mathrm{F}}}F^{\top}F\right)