Repository: explosion/spaCy

Issue:
problem training text categorizer on GPU 
Great work!
Install spacy in an isolated virtualenv using `pip install spacy[cuda90]`, everything works well and 
training starts normally but when GPU memory starts to be used as reported by `nvidia-smi` I get this error and the process ends, the environment is well configured and I'm able to use my GPUs with tensor flow.
```
Traceback (most recent call last):
  File "core_nlp/spacy/text_categorizer.py", line 168, in <module>
    nlp = TextCategorizer.train(filter_by_support(category_data, 60), model, None, use_averages=True, epochs=50, debug=False)
  File "core_nlp/spacy/text_categorizer.py", line 80, in train
    nlp_.update(texts, annotations, sgd=optimizer, drop=dropout, losses=losses)
  File "/home/neo/pyenvs/spacyp3/lib/python3.5/site-packages/spacy/language.py", line 421, in update
    proc.update(docs, golds, drop=drop, sgd=get_grads, losses=losses)
  File "pipeline.pyx", line 876, in spacy.pipeline.TextCategorizer.update
  File "/home/neo/pyenvs/spacyp3/lib/python3.5/site-packages/thinc/api.py", line 61, in begin_update
    X, inc_layer_grad = layer.begin_update(X, drop=drop)
  File "/home/neo/pyenvs/spacyp3/lib/python3.5/site-packages/thinc/api.py", line 176, in begin_update
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/home/neo/pyenvs/spacyp3/lib/python3.5/site-packages/thinc/api.py", line 176, in <listcomp>
    values = [fwd(X, *a, **k) for fwd in forward]
  File "/home/neo/pyenvs/spacyp3/lib/python3.5/site-packages/thinc/api.py", line 258, in wrap
    output = func(*args, **kwargs)
  File "/home/neo/pyenvs/spacyp3/lib/python3.5/site-packages/thinc/api.py", line 61, in begin_update
    X, inc_layer_grad = layer.begin_update(X, drop=drop)
  File "/home/neo/pyenvs/spacyp3/lib/python3.5/site-packages/spacy/_ml.py", line 102, in _preprocess_doc
    keys = ops.xp.concatenate(keys)
  File "/home/neo/pyenvs/spacyp3/lib/python3.5/site-packages/cupy/manipulation/join.py", line 49, in concatenate
    return core.concatenate_method(tup, axis)
  File "cupy/core/core.pyx", line 2740, in cupy.core.core.concatenate_method
  File "cupy/core/core.pyx", line 2753, in cupy.core.core.concatenate_method
TypeError: Only cupy arrays can be concatenated
```
This is a copy of my requirements.txt
```
certifi==2018.10.15
chardet==3.0.4
cupy-cuda90==5.0.0
cycler==0.10.0
cymem==2.0.2
cytoolz==0.9.0.1
dill==0.2.8.2
fastrlock==0.4
idna==2.7
kiwisolver==1.0.1
matplotlib==3.0.2
msgpack==0.5.6
msgpack-numpy==0.4.3.2
murmurhash==1.0.1
numpy==1.15.4
pandas==0.23.4
plac==0.9.6
preshed==2.0.1
pyparsing==2.3.0
python-dateutil==2.7.5
pytz==2018.7
regex==2018.1.10
requests==2.20.1
scikit-learn==0.20.0
scipy==1.1.0
six==1.11.0
spacy==2.0.16
thinc==6.12.0
thinc-gpu-ops==0.0.3
toolz==0.9.0
tqdm==4.28.1
twitter-text-python==1.1.0
ujson==1.35
urllib3==1.24.1
wrapt==1.10.11
```
## Your Environment
<!-- Include details of your environment. If you're using spaCy 1.7+, you can also type `python -m spacy info --markdown` and copy-paste the result here.-->
* Operating System: Ubuntu 16.04
* Python Version Used: 3.5.2
* spaCy Version Used: 2.0.16
* Environment Information: Cuda 90 driver working with tensor flow


Discussion:

Resolving patch:
diff --git a/examples/training/train_textcat.py b/examples/training/train_textcat.py
--- a/examples/training/train_textcat.py
+++ b/examples/training/train_textcat.py
@@ -44,6 +44,7 @@ def main(model=None, output_dir=None, n_iter=20, n_texts=2000):
 
     # add label to text classifier
     textcat.add_label("POSITIVE")
+    textcat.add_label("NEGATIVE")
 
     # load the IMDB dataset
     print("Loading IMDB data...")
@@ -64,7 +65,7 @@ def main(model=None, output_dir=None, n_iter=20, n_texts=2000):
         for i in range(n_iter):
             losses = {}
             # batch up the examples using spaCy's minibatch
-            batches = minibatch(train_data, size=compounding(4.0, 32.0, 1.001))
+            batches = minibatch(train_data, size=compounding(4.0, 16.0, 1.001))
             for batch in batches:
                 texts, annotations = zip(*batch)
                 nlp.update(texts, annotations, sgd=optimizer, drop=0.2, losses=losses)
@@ -106,22 +107,24 @@ def load_data(limit=0, split=0.8):
     random.shuffle(train_data)
     train_data = train_data[-limit:]
     texts, labels = zip(*train_data)
-    cats = [{"POSITIVE": bool(y)} for y in labels]
+    cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in labels]
     split = int(len(train_data) * split)
     return (texts[:split], cats[:split]), (texts[split:], cats[split:])
 
 
 def evaluate(tokenizer, textcat, texts, cats):
     docs = (tokenizer(text) for text in texts)
-    tp = 1e-8  # True positives
+    tp = 0.0  # True positives
     fp = 1e-8  # False positives
     fn = 1e-8  # False negatives
-    tn = 1e-8  # True negatives
+    tn = 0.0  # True negatives
     for i, doc in enumerate(textcat.pipe(docs)):
         gold = cats[i]
         for label, score in doc.cats.items():
             if label not in gold:
                 continue
+            if label == "NEGATIVE":
+                continue
             if score >= 0.5 and gold[label] >= 0.5:
                 tp += 1.0
             elif score >= 0.5 and gold[label] < 0.5:
diff --git a/spacy/_ml.py b/spacy/_ml.py
--- a/spacy/_ml.py
+++ b/spacy/_ml.py
@@ -5,7 +5,7 @@
 from thinc.v2v import Model, Maxout, Softmax, Affine, ReLu
 from thinc.i2v import HashEmbed, StaticVectors
 from thinc.t2t import ExtractWindow, ParametricAttention
-from thinc.t2v import Pooling, sum_pool
+from thinc.t2v import Pooling, sum_pool, mean_pool
 from thinc.misc import Residual
 from thinc.misc import LayerNorm as LN
 from thinc.misc import FeatureExtracter
@@ -575,6 +575,32 @@ def build_text_classifier(nr_class, width=64, **cfg):
     return model
 
 
+def build_simple_cnn_text_classifier(tok2vec, nr_class, exclusive_classes=True, **cfg):
+    """
+    Build a simple CNN text classifier, given a token-to-vector model as inputs.
+    If exclusive_classes=True, a softmax non-linearity is applied, so that the
+    outputs sum to 1. If exclusive_classes=False, a logistic non-linearity
+    is applied instead, so that outputs are in the range [0, 1].
+    """
+    with Model.define_operators({">>": chain}):
+        if exclusive_classes:
+            output_layer = Softmax(nr_class, tok2vec.nO)
+        else:
+            output_layer = (
+                zero_init(Affine(nr_class, tok2vec.nO))
+                >> logistic
+            )
+        model = (
+            tok2vec
+            >> flatten_add_lengths
+            >> Pooling(mean_pool)
+            >> output_layer
+        )
+    model.tok2vec = chain(tok2vec, flatten)
+    model.nO = nr_class
+    return model
+
+
 @layerize
 def flatten(seqs, drop=0.0):
     ops = Model.ops


Test patch: