Repository: explosion/thinc

Issue:
Thinc GPU capability on Windows - fails to compile
It looks like Thinc has some hard time making GPU computation available if used on Windows.

Similar issues were raised in spaCy and thinc:
https://github.com/explosion/spaCy/issues/1745
https://github.com/explosion/thinc/issues/50

It seems related in how the build is set-up in setup.py:
https://github.com/explosion/thinc/blob/master/setup.py#L56

Based on quick search, others are struggling with this as well:
https://github.com/rbgirshick/py-faster-rcnn/issues/11

Some more info (they came to a working solution there):
https://github.com/cudamat/cudamat/issues/46#issuecomment-73803650

Will research a bit more later.


Discussion:
Sorry this is hard. I don't use Windows, so am not of much help. You might try the wheels here? https://www.lfd.uci.edu/~gohlke/pythonlibs/ Some advertise CUDA linkage. It might solve your problem.
Has there been a resolution to this? Right now using linux through VM so I do not have access to GPU. Windows solution would be extremely welcome...
I'm also having trouble getting GPU working on Windows. I'm not sure what issue you are having, but in my case, it's the fact that `thinc.neural.gpu_ops` isn't getting created.

I should note that I'm trying to do this using CUDA v9, so some of the issue may be due to that. Also, I just started digging into this today, so I may be way off.

I think the issue is in the locate_cuda function in setup.py. I think there are two separate issue with the function.

First, the following snippet fails to see nvcc, even though it's there.
```
for k, v in cudaconfig.items():
    if not os.path.exists(v):
        print('Warning: The CUDA %s path could not be located in %s' % (k, v))
        return None
```
I think this is because
`os.path.exists('[CUDA_HOME]\bin\nvcc)` returns `False`,
but
`os.path.exists('[CUDA_HOME]\bin\nvcc.exe)` returns `True`.

I'm guessing this is a platform dependent issue.

Also, the code is looking for a `lib64` directory under the CUDA_HOME directory. In my case, this doesn't exist, though there is a `lib` directory with an `x64` directory under that. I assume that is the directory it should be pointing to. I'm not sure whether this is a platform specfic issue or an issue due to the fact that I'm using CUDA v9.

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -228,12 +228,13 @@ def setup_package():
                 'pathlib==1.0.1; python_version < "3.4"',
             ],
             extras_require={
-                "cuda": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy>=5.0.0b4"],
-                "cuda80": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda80>=5.0.0b4"],
-                "cuda90": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda90>=5.0.0b4"],
-                "cuda91": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda91>=5.0.0b4"],
-                "cuda92": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda92>=5.0.0b4"],
-                "cuda100": ["thinc_gpu_ops>=0.0.1,<0.1.0", "cupy-cuda100>=5.0.0b4"],
+                "cuda": ["cupy>=5.0.0b4"],
+                "cuda80": ["cupy-cuda80>=5.0.0b4"],
+                "cuda90": ["cupy-cuda90>=5.0.0b4"],
+                "cuda91": ["cupy-cuda91>=5.0.0b4"],
+                "cuda92": ["cupy-cuda92>=5.0.0b4"],
+                "cuda100": ["cupy-cuda100>=5.0.0b4"],
+                "cuda110": ["cupy-cuda110>=5.0.0b4"],
             },
             classifiers=[
                 "Development Status :: 5 - Production/Stable",
diff --git a/thinc/neural/_custom_kernels.py b/thinc/neural/_custom_kernels.py
new file mode 100644
--- /dev/null
+++ b/thinc/neural/_custom_kernels.py
@@ -0,0 +1,131 @@
+from __future__ import unicode_literals
+from numpy.testing import assert_allclose
+import re
+from pathlib import Path
+from collections import defaultdict
+
+try:
+    import cupy
+except ImportError:
+    cupy = None
+
+
+kernel_re = re.compile(r"extern \"C\" __global__.+?(?=extern|$)", re.DOTALL)
+name_re = re.compile(r"(?<=void )\w+(?=\()")
+def parse_kernels(src):
+    kernels = {}
+    for kernel in kernel_re.findall(src):
+        name = name_re.search(kernel).group()
+        kernels[name] = kernel
+    return kernels
+
+
+def compile_kernels(src):
+    if cupy is None:
+        return defaultdict(lambda: None)
+    kernels = parse_kernels(src)
+    return {name: cupy.RawKernel(src, name) for name, src in kernels.items()}
+
+def compile_mmh(src):
+    if cupy is None:
+        return None
+    return cupy.RawKernel(src, "hash_data")
+
+    
+PWD = Path(__file__).parent
+SRC = (PWD / "_custom_kernels.cu").open("r", encoding="utf8").read()
+KERNELS = compile_kernels(SRC)
+
+MMH_SRC = (PWD / "_murmur3.cu").open("r", encoding="utf8").read()
+KERNELS["hash"] = compile_mmh(MMH_SRC)
+
+sum_pool_kernel = KERNELS["sum_pool"]
+max_pool_kernel = KERNELS["max_pool"]
+maxout_kernel = KERNELS["maxout"]
+backprop_sum_pool_kernel = KERNELS["backprop_sum_pool"]
+backprop_max_pool_kernel = KERNELS["backprop_max_pool"]
+hash_data_kernel = compile_mmh(MMH_SRC)
+
+
+def sum_pool(X, lengths, out=None, threads_per_block=128):
+    if out is None:
+        out = cupy.zeros((len(lengths), X.shape[1]), dtype="f")
+    B = len(lengths)
+    T = X.shape[0]
+    O = X.shape[1]
+    num_blocks = min(1, B // threads_per_block)
+    sum_pool_kernel((num_blocks,), (threads_per_block,), (out, X, lengths, B, T, O))
+    return out
+
+
+def mean_pool(X, lengths, out=None, threads_per_block=128):
+    if out is None:
+        out = cupy.zeros((len(lengths), X.shape[1]), dtype="f")
+    B = len(lengths)
+    T = X.shape[0]
+    O = X.shape[1]
+    num_blocks = min(1, B // threads_per_block)
+    sum_pool_kernel((num_blocks,), (threads_per_block,), (out, X, lengths, B, T, O))
+    out /= lengths
+    return out
+
+
+def max_pool(X, lengths, out=None, threads_per_block=128):
+    if out is None:
+        maxes = cupy.zeros((len(lengths), X.shape[1]), dtype="f")
+        which = cupy.zeros((len(lengths), X.shape[1]), dtype="i")
+    else:
+        maxes, which = out
+    B = len(lengths)
+    T = X.shape[0]
+    O = X.shape[1]
+    num_blocks = max(1, B // threads_per_block)
+    max_pool_kernel((num_blocks,), (threads_per_block,),
+        (maxes, which, X, lengths, B, T, O))
+    return maxes, which
+
+
+def backprop_sum_pool(d_sum, lengths, out=None, threads_per_block=128):
+    B = len(lengths)
+    T = int(lengths.sum())
+    O = d_sum.shape[1]
+    if out is None:
+        out = cupy.zeros((T, O), dtype="f")
+
+    num_blocks = max(1, T // threads_per_block)
+    backprop_sum_pool_kernel((num_blocks,), (threads_per_block,),
+        (out, d_sum, lengths, B, T, O))
+    return out
+
+
+def backprop_mean_pool(d_mean, lengths, out=None, threads_per_block=128):
+    out = backprop_sum_pool(d_mean, lengths, out=out,
+            threads_per_block=threads_per_block)
+    out /= lengths
+    return out
+
+
+def backprop_max_pool(d_maxes, which, lengths, out=None, threads_per_block=128):
+    B = len(lengths)
+    T = int(lengths.sum())
+    O = d_maxes.shape[1]
+    if out is None:
+        out = cupy.zeros((T, O), dtype="f")
+
+    num_blocks = max(1, T // threads_per_block)
+    backprop_max_pool_kernel((num_blocks,), (threads_per_block,),
+        (out, d_maxes, which, lengths, B, T, O))
+    return out
+
+
+def hash(ids, seed, out=None, threads_per_block=128):
+    if out is None:
+        out = cupy.zeros((ids.shape[0], 4), dtype="uint32")
+    # sizeof(uint32_t) * 4
+    out_size = 4 * 4
+    in_size = 8 # sizeof(uint64_t)
+    T = ids.shape[0]
+    # Having trouble executing this in parallel? Shrug
+    hash_data_kernel((T,), (1,),
+        (out, ids, out_size, in_size, ids.shape[0], seed))
+    return out


Test patch:
diff --git a/thinc/tests/unit/test_ops.py b/thinc/tests/unit/test_ops.py
--- a/thinc/tests/unit/test_ops.py
+++ b/thinc/tests/unit/test_ops.py
@@ -154,24 +154,41 @@ def finish_update(d, *args, **kwargs):
             assert output_gradient[i, j] == 1.0 * (4.0 / 3.0)
 
 
+@settings(max_examples=MAX_EXAMPLES)
+@given(X=strategies.arrays_BI())
+def test_backprop_sum_pool(ops, X):
+    X = ops.asarray(X)
+    if ops.xp.abs(X).max() >= 30:
+        return None
+    lengths = ops.asarray([3] * len(X), dtype="i")
+    out = ops.backprop_sum_pool(X, lengths)
+    assert out.shape == (sum(lengths), X.shape[1])
+    start = 0
+    for i, length in enumerate(lengths):
+        ops.xp.testing.assert_allclose(out[start : length].sum(axis=0), X[i] * length)
+
+
+
 @settings(max_examples=MAX_EXAMPLES)
 @given(X=strategies.arrays_BI())
 def test_softmax_sums_to_one(ops, X):
     y = ops.softmax(ops.asarray(X))
     for row in y:
-        assert 0.99999 <= row.sum() <= 1.00001
+        assert 0.99999 <= row.sum() <= 1.0001
 
 
 @settings(max_examples=MAX_EXAMPLES)
 @given(X=strategies.arrays_BI())
 def test_softmax_sequence_sums_to_two(ops, X):
+    X = ops.asarray(X)
+    if ops.xp.abs(X).max() >= 30:
+        return None
     half = X.shape[0] // 2
     if half >= 1:
-        X = ops.asarray(X)
         lengths = ops.asarray([half, X.shape[0] - half], dtype="i")
         y = ops.softmax_sequences(X, lengths)
         for col in y.sum(axis=0):
-            assert 0.99999 <= col <= 2.00001
+            assert 0.99999 <= col <= 2.0001, col
 
 
 @settings(max_examples=MAX_EXAMPLES)
@@ -268,3 +285,26 @@ def test_flatten_unflatten_roundtrip(cpu_ops, X):
     assert flat.ndim == 1
     unflat = cpu_ops.unflatten(flat, [len(x) for x in X])
     assert_allclose(X, unflat)
+
+
+def test_sum_pool(ops):
+    m = ops.xp.zeros((19, 5), dtype="f")
+    m += 1
+    lengths = ops.xp.array([5,5,3,6], dtype="i")
+    output = ops.sum_pool(m, lengths)
+    assert output.sum() == m.sum(), (output.sum(), m.sum())
+
+
+def test_max_pool(ops):
+    m = ops.xp.zeros((19, 5), dtype="f")
+    m += ops.xp.random.uniform(-1, 1, m.shape)
+    lengths = ops.xp.array([5,5,3,6], dtype="i")
+    m[4, 0] = 1
+    m[0, 1] = 2
+    m[1, 3] = 3
+    maxes, which = ops.max_pool(m, lengths)
+    start = 0
+    for i, length in enumerate(lengths):
+        truth = m[start:start+length].max(axis=0)
+        ops.xp.testing.assert_allclose(maxes[i], truth)
+        start += length