Repository: explosion/thinc

Issue:
ExtractWindow for nW > 1 gives wrong output ?
On thinc==6.10.1,

```python
data = [[1], [2], [3]]
d = np.array(data, dtype=np.float32)
e = ExtractWindow(nW=1)
print e(d)
```

which gives correct output as expected

```
array([[ 0.,  1.,  2.],
       [ 1.,  2.,  3.],
       [ 2.,  3.,  0.]], dtype=float32)
```

But 
```python
data = [[1], [2], [3]]
d = np.array(data, dtype=np.float32)
e = ExtractWindow(nW=2)
print e(d)
```

outputs

```
array([[ 0.,  0.,  1.,  2.,  3.],
       [ 1.,  2.,  2.,  3.,  0.],
       [ 0.,  0.,  0.,  0.,  0.]], dtype=float32)
```

If I understood the purpose of the method - concat by sliding a window, shouldn't this be

```
array([[ 0.,  0.,  1.,  2.,  3.],
       [ 0.,  1.,  2.,  3.,  0.],
       [ 1.,  2.,  3.,  0.,  0.]], dtype=float32)
```


Discussion:
Definitely looks wrong --- thanks!

The problem will almost surely be in the `NumpyOps.seq2col` and `CupyOps.seq2col` methods.
I just tried, it seems the bug stll exists in 7.0.8 version. I think this might hinder users customize their own models

Resolving patch:
diff --git a/thinc/neural/_custom_kernels.py b/thinc/neural/_custom_kernels.py
--- a/thinc/neural/_custom_kernels.py
+++ b/thinc/neural/_custom_kernels.py
@@ -39,38 +39,47 @@ def compile_mmh(src):
 MMH_SRC = (PWD / "_murmur3.cu").open("r", encoding="utf8").read()
 KERNELS["hash"] = compile_mmh(MMH_SRC)
 
+seq2col_kernel = KERNELS["seq2col"]
 sum_pool_kernel = KERNELS["sum_pool"]
 max_pool_kernel = KERNELS["max_pool"]
 maxout_kernel = KERNELS["maxout"]
+backprop_seq2col_kernel = KERNELS["backprop_seq2col"]
 backprop_sum_pool_kernel = KERNELS["backprop_sum_pool"]
 backprop_max_pool_kernel = KERNELS["backprop_max_pool"]
 hash_data_kernel = compile_mmh(MMH_SRC)
 
 
-def sum_pool(X, lengths, out=None, threads_per_block=128):
+def seq2col(X, nW, out=None, threads_per_block=128, num_blocks=128):
+    if out is None:
+        out = cupy.zeros((X.shape[0], X.shape[1] * ((nW*2)+1)), dtype="f")
+    B = X.shape[0]
+    I = X.shape[1]
+    seq2col_kernel((num_blocks,), (threads_per_block,), (out, X, nW, B, I))
+    return out
+
+
+def sum_pool(X, lengths, out=None, threads_per_block=128, num_blocks=128):
     if out is None:
         out = cupy.zeros((len(lengths), X.shape[1]), dtype="f")
     B = len(lengths)
     T = X.shape[0]
     O = X.shape[1]
-    num_blocks = min(1, B // threads_per_block)
     sum_pool_kernel((num_blocks,), (threads_per_block,), (out, X, lengths, B, T, O))
     return out
 
 
-def mean_pool(X, lengths, out=None, threads_per_block=128):
+def mean_pool(X, lengths, out=None, threads_per_block=128, num_blocks=128):
     if out is None:
         out = cupy.zeros((len(lengths), X.shape[1]), dtype="f")
     B = len(lengths)
     T = X.shape[0]
     O = X.shape[1]
-    num_blocks = min(1, B // threads_per_block)
     sum_pool_kernel((num_blocks,), (threads_per_block,), (out, X, lengths, B, T, O))
     out /= lengths
     return out
 
 
-def max_pool(X, lengths, out=None, threads_per_block=128):
+def max_pool(X, lengths, out=None, threads_per_block=128, num_blocks=128):
     if out is None:
         maxes = cupy.zeros((len(lengths), X.shape[1]), dtype="f")
         which = cupy.zeros((len(lengths), X.shape[1]), dtype="i")
@@ -79,46 +88,55 @@ def max_pool(X, lengths, out=None, threads_per_block=128):
     B = len(lengths)
     T = X.shape[0]
     O = X.shape[1]
-    num_blocks = max(1, B // threads_per_block)
     max_pool_kernel((num_blocks,), (threads_per_block,),
         (maxes, which, X, lengths, B, T, O))
     return maxes, which
 
 
-def backprop_sum_pool(d_sum, lengths, out=None, threads_per_block=128):
+def backprop_seq2col(dY, nW, out=None, threads_per_block=128, num_blocks=128):
+    B = dY.shape[0]
+    nF = nW*2+1
+    I = dY.shape[1] // nF
+    if out is None:
+        out = cupy.zeros((B, I), dtype="f")
+    backprop_seq2col_kernel((num_blocks,), (threads_per_block,),
+        (out, dY, nW, B, I))
+    return out
+
+
+def backprop_sum_pool(d_sum, lengths, out=None, threads_per_block=128, num_blocks=128):
     B = len(lengths)
     T = int(lengths.sum())
     O = d_sum.shape[1]
     if out is None:
         out = cupy.zeros((T, O), dtype="f")
 
-    num_blocks = max(1, T // threads_per_block)
     backprop_sum_pool_kernel((num_blocks,), (threads_per_block,),
         (out, d_sum, lengths, B, T, O))
     return out
 
 
-def backprop_mean_pool(d_mean, lengths, out=None, threads_per_block=128):
+def backprop_mean_pool(d_mean, lengths, out=None, threads_per_block=128, num_blocks=128):
     out = backprop_sum_pool(d_mean, lengths, out=out,
-            threads_per_block=threads_per_block)
+            threads_per_block=threads_per_block, num_blocks=num_blocks)
     out /= lengths
     return out
 
 
-def backprop_max_pool(d_maxes, which, lengths, out=None, threads_per_block=128):
+def backprop_max_pool(d_maxes, which, lengths, out=None,
+        threads_per_block=128, num_blocks=128):
     B = len(lengths)
     T = int(lengths.sum())
     O = d_maxes.shape[1]
     if out is None:
         out = cupy.zeros((T, O), dtype="f")
 
-    num_blocks = max(1, T // threads_per_block)
     backprop_max_pool_kernel((num_blocks,), (threads_per_block,),
         (out, d_maxes, which, lengths, B, T, O))
     return out
 
 
-def hash(ids, seed, out=None, threads_per_block=128):
+def hash(ids, seed, out=None, threads_per_block=128, num_blocks=128):
     if out is None:
         out = cupy.zeros((ids.shape[0], 4), dtype="uint32")
     # sizeof(uint32_t) * 4
@@ -126,6 +144,6 @@ def hash(ids, seed, out=None, threads_per_block=128):
     in_size = 8 # sizeof(uint64_t)
     T = ids.shape[0]
     # Having trouble executing this in parallel? Shrug
-    hash_data_kernel((T,), (1,),
+    hash_data_kernel((num_blocks,), (threads_per_block,),
         (out, ids, out_size, in_size, ids.shape[0], seed))
     return out


Test patch:
diff --git a/thinc/tests/unit/test_ops.py b/thinc/tests/unit/test_ops.py
--- a/thinc/tests/unit/test_ops.py
+++ b/thinc/tests/unit/test_ops.py
@@ -7,7 +7,7 @@
 from numpy.testing import assert_allclose
 
 from .. import strategies
-from ...neural.ops import NumpyOps, CupyOps
+from ...neural.ops import NumpyOps, CupyOps, Ops
 
 
 MAX_EXAMPLES = 10
@@ -60,7 +60,7 @@ def test_get_dropout_not_empty(ops):
     assert all(value >= 0 for value in mask.flatten())
 
 
-def test_seq2col_window_one(ops):
+def test_seq2col_window_one_small(ops):
     seq = ops.asarray([[1.0], [3.0], [4.0], [5]], dtype="float32")
     cols = ops.seq2col(seq, 1)
     if not isinstance(cols, numpy.ndarray):
@@ -70,8 +70,18 @@ def test_seq2col_window_one(ops):
     assert_allclose(cols[2], [3.0, 4.0, 5.0])
     assert_allclose(cols[3], [4.0, 5.0, 0.0])
 
+@settings(max_examples=MAX_EXAMPLES)
+@given(X=strategies.arrays_BI())
+def test_seq2col_window_one(ops, X):
+    X = ops.asarray(X)
+    base_ops = Ops()
+    base_ops.xp = ops.xp
+    target = base_ops.seq2col(X, nW=1)
+    predicted = ops.seq2col(X, nW=1)
+    ops.xp.testing.assert_allclose(target, predicted)
+
 
-def test_backprop_seq2col_window_one(ops):
+def test_backprop_seq2col_window_one_small(ops):
     cols = ops.asarray(
         [[0.0, 0.0, 0.0], [-1.0, 0.0, 1.0], [2.0, 0.0, 0.0]], dtype="float32"
     )
@@ -81,8 +91,28 @@ def test_backprop_seq2col_window_one(ops):
         seq = seq.get()
     assert_allclose(seq, expected)
 
+@settings(max_examples=MAX_EXAMPLES)
+@given(X=strategies.arrays_BI())
+def test_backprop_seq2col_window_one(ops, X):
+    if X.shape[1] % 3:
+        return None
+    X = ops.asarray(X)
+    if ops.xp.abs(X).max() >= 30:
+        return None
+    base_ops = Ops()
+    base_ops.xp = ops.xp
+    target = base_ops.backprop_seq2col(X, nW=1)
+    predicted = ops.backprop_seq2col(X, nW=1)
+    for row in range(target.shape[0]):
+        diff = target[row].sum() - predicted[row].sum()
+        if diff < -0.1 or diff > 0.1:
+            print(row, diff)
+            print(target[row])
+            print(predicted[row])
+    ops.xp.testing.assert_allclose(target, predicted)
+
+
 
-@pytest.mark.xfail
 def test_seq2col_window_two(ops):
     seq = ops.asarray([[1.0], [2.0], [3.0], [4]], dtype="float32")
     cols = ops.seq2col(seq, 2)
@@ -94,18 +124,25 @@ def test_seq2col_window_two(ops):
     assert_allclose(cols[3], [2.0, 3.0, 4.0, 0.0, 0.0])
 
 
-# def test_backprop_seq2col_window_two(ops):
-#    cols = ops.asarray([
-#        [0., 0., 0.],
-#        [-1., 0., 1.],
-#        [2., 0., 0.],
-#    ], dtype='float32')
-#    expected = [[-1.], [2.], [1.]]
-#    seq = ops.backprop_seq2col(cols, 1)
-#    if not isinstance(seq, numpy.ndarray):
-#        seq = seq.get()
-#    assert_allclose(seq, expected)
-#
+def test_backprop_seq2col_window_two(ops):
+    cols = ops.asarray([
+        [0.0, 0.0, 1.0, 2.0, 3.0],
+        [0.0, 1.0, 2.0, 3.0, 4.0],
+        [1.0, 2.0, 3.0, 4.0, 0.0],
+        [2.0, 3.0, 4.0, 0.0, 0.0]
+    ], dtype="float32")
+    # We're summing the values that each row
+    # was used as a feature. So row 0 had a
+    # gradient of 1 in row 0, 1 in row 2, and
+    # 1 in row 3. 
+    expected = ops.asarray([
+        [1+1+1.+0.],
+        [2.+2.+2.+2.],
+        [3.+3.+3.+3.],
+        [0.+4.+4.+4.]
+    ], dtype="f")
+    seq = ops.backprop_seq2col(cols, 2)
+    ops.xp.testing.assert_allclose(seq, expected)
 
 
 @settings(max_examples=MAX_EXAMPLES)