Repository: HIPS/autograd (Python)

Issue:
linalg.norm has mismatched convention for complex numbers
Hi all, thanks for such a still useful project.

The convention adopted in `autograd` for complex numbers is generally that parameters can be updated in the direction of `conj(g)` to minimize a function. `linalg.norm` follows the opposite convention, preventing it from being used in code that combines it with other functions.

This was mentioned in #393 alongside some potential fixes.

Minimal reproducer:
```python
import numpy as np
import autograd.numpy as anp
from autograd import value_and_grad

rng = np.random.default_rng(42)

x = rng.normal(size=10) + 1j * rng.normal(10)
y = rng.normal(size=10) + 1j * rng.normal(10)

def foo(x, y):
    # simple loss function
    return anp.linalg.norm(x - y)

def foo2(x, y):
    # alt impl
    d = x - y
    return anp.sum(anp.abs(d)**2)**0.5

# use foo2 for correct behavior
foo_vag = value_and_grad(foo)

for _ in range(10):
    v, g = foo_vag(x, y)
    # or remove conj here to make foo work
    x -= 0.05 * np.conj(g)
    # value should initially decrease
    print(v)
# 5.761837589211455
# 5.785341294273737
# 5.8095148223718915
# 5.834344369254469
# 5.8598161062729055
# 5.885916205539433
# 5.912630863722038
# 5.939946324450408
# 5.967848899318491
# 5.996324987479975
```
autograd v1.7.0.


Discussion:
Thanks for the report, @jcmgray! If #393 is a complete fix, we could merge it. It does look like nothing too invasive, so I'll rebase that branch and check.
#393 doesn't actually fix this issue; it only comments on it.

Opened a PR #705 for a fix.



Patch:
diff --git a/autograd/numpy/linalg.py b/autograd/numpy/linalg.py
index b400a558..4ed0b1ff 100644
--- a/autograd/numpy/linalg.py
+++ b/autograd/numpy/linalg.py
@@ -112,7 +112,7 @@ def check_implemented():
 
     def vjp(g):
         if ord in (None, 2, "fro"):
-            return expand(g / ans) * x
+            return expand(g / ans) * anp.conj(x)
         elif ord == "nuc":
             x_rolled = roll(x)
             u, s, vt = svd(x_rolled, full_matrices=False)
@@ -120,10 +120,10 @@ def vjp(g):
             # Roll the matrix axes back to their correct positions
             uvt = unroll(uvt_rolled)
             g = expand(g)
-            return g * uvt
+            return g * anp.conj(uvt)
         else:
             # see https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm
-            return expand(g / ans ** (ord - 1)) * x * anp.abs(x) ** (ord - 2)
+            return expand(g / ans ** (ord - 1)) * anp.conj(x) * anp.abs(x) ** (ord - 2)
 
     return vjp
 
@@ -161,17 +161,17 @@ def check_implemented():
 
     check_implemented()
     if ord in (None, 2, "fro"):
-        return contract(g * x) / ans
+        return contract(g * anp.conj(x)) / ans
     elif ord == "nuc":
         x_rolled = roll(x)
         u, s, vt = svd(x_rolled, full_matrices=False)
         uvt_rolled = _dot(u, vt)
         # Roll the matrix axes back to their correct positions
         uvt = unroll(uvt_rolled)
-        return contract(g * uvt)
+        return contract(g * anp.conj(uvt))
     else:
         # see https://en.wikipedia.org/wiki/Norm_(mathematics)#p-norm
-        return contract(g * x * anp.abs(x) ** (ord - 2)) / ans ** (ord - 1)
+        return contract(g * anp.conj(x) * anp.abs(x) ** (ord - 2)) / ans ** (ord - 1)
 
 
 defjvp(norm, norm_jvp)


Test patch:
diff --git a/tests/test_linalg.py b/tests/test_linalg.py
index 3bf540a2..934ceabc 100644
--- a/tests/test_linalg.py
+++ b/tests/test_linalg.py
@@ -175,6 +175,15 @@ def fun(x):
     check_grads(fun, modes=["fwd", "rev"])(vec)
 
 
+def test_vector_2norm_complex():
+    def fun(x):
+        return np.linalg.norm(x)
+
+    D = 6
+    vec = npr.randn(D) + 1j * npr.randn(D)
+    check_grads(fun)(vec)
+
+
 def test_frobenius_norm():
     def fun(x):
         return np.linalg.norm(x)
@@ -184,6 +193,15 @@ def fun(x):
     check_grads(fun, modes=["fwd", "rev"])(mat)
 
 
+def test_frobenius_norm_complex():
+    def fun(x):
+        return np.linalg.norm(x)
+
+    D = 6
+    mat = npr.randn(D, D - 1) + 1j * npr.randn(D, D - 1)
+    check_grads(fun)(mat)
+
+
 def test_frobenius_norm_axis():
     def fun(x):
         return np.linalg.norm(x, axis=(0, 1))
@@ -193,6 +211,15 @@ def fun(x):
     check_grads(fun, modes=["fwd", "rev"])(mat)
 
 
+def test_frobenius_norm_axis_complex():
+    def fun(x):
+        return np.linalg.norm(x, axis=(0, 1))
+
+    D = 6
+    mat = npr.randn(D, D - 1, D - 2) + 1j * npr.randn(D, D - 1, D - 2)
+    check_grads(fun)(mat)
+
+
 @pytest.mark.parametrize("ord", range(2, 5))
 @pytest.mark.parametrize("size", [6])
 def test_vector_norm_ord(size, ord):
@@ -203,6 +230,16 @@ def fun(x):
     check_grads(fun, modes=["fwd", "rev"])(vec)
 
 
+@pytest.mark.parametrize("ord", range(2, 5))
+@pytest.mark.parametrize("size", [6])
+def test_vector_norm_ord_complex(size, ord):
+    def fun(x):
+        return np.linalg.norm(x, ord=ord)
+
+    vec = npr.randn(size) + 1j * npr.randn(size)
+    check_grads(fun)(vec)
+
+
 @pytest.mark.parametrize("axis", range(3))
 @pytest.mark.parametrize("shape", [(6, 5, 4)])
 def test_norm_axis(shape, axis):
@@ -213,6 +250,16 @@ def fun(x):
     check_grads(fun, modes=["fwd", "rev"])(arr)
 
 
+@pytest.mark.parametrize("axis", range(3))
+@pytest.mark.parametrize("shape", [(6, 5, 4)])
+def test_norm_axis_complex(shape, axis):
+    def fun(x):
+        return np.linalg.norm(x, axis=axis)
+
+    arr = npr.randn(*shape) + 1j * npr.randn(*shape)
+    check_grads(fun)(arr)
+
+
 def test_norm_nuclear():
     def fun(x):
         return np.linalg.norm(x, ord="nuc")
@@ -223,6 +270,15 @@ def fun(x):
     check_grads(fun, modes=["fwd", "rev"], order=1)(mat)
 
 
+def test_norm_nuclear_complex():
+    def fun(x):
+        return np.linalg.norm(x, ord="nuc")
+
+    D = 6
+    mat = npr.randn(D, D - 1) + 1j * npr.randn(D, D - 1)
+    check_grads(fun)(mat)
+
+
 def test_norm_nuclear_axis():
     def fun(x):
         return np.linalg.norm(x, ord="nuc", axis=(0, 1))
@@ -233,6 +289,15 @@ def fun(x):
     check_grads(fun, modes=["fwd", "rev"], order=1)(mat)
 
 
+def test_norm_nuclear_axis_complex():
+    def fun(x):
+        return np.linalg.norm(x, ord="nuc", axis=(0, 1))
+
+    D = 6
+    mat = npr.randn(D, D - 1, D - 2) + 1j * npr.randn(D, D - 1, D - 2)
+    check_grads(fun)(mat)
+
+
 def test_eigvalh_lower():
     def fun(x):
         w, v = np.linalg.eigh(x)