Repository: explosion/spaCy

Issue:
Pretraining: batch size & document length
## Feature description

I was just testing the `pretrain` command and ran out of memory ^^ !
Looking at the code, it seems that the `batch_size` is fixed to 3000. I also noticed that document with length greater than 500 token are pruned by default. I guess this are default parameters for a corpus of sentence / small text... But maybe we should let the user set these parameters ?




Discussion:
I was looking into pretraining as well, since I only have a very limited number of labeled data and was hoping to boost the performance.  
Would it make sense to split the data in single sentences or maybe n-gram sentences?
Try splitting into paragraphs or sentences at the moment, yeah. The `pretrain` command is still a bit rough in these respects --- it'll continue to improve.

Resolving patch:
diff --git a/spacy/cli/pretrain.py b/spacy/cli/pretrain.py
--- a/spacy/cli/pretrain.py
+++ b/spacy/cli/pretrain.py
@@ -29,6 +29,9 @@
     embed_rows=("Embedding rows", "option", "er", int),
     use_vectors=("Whether to use the static vectors as input features", "flag", "uv"),
     dropout=("Dropout", "option", "d", float),
+    batch_size=("Number of words per training batch", "option", "bs", int),
+    max_length=("Max words per example.", "option", "xw", int),
+    min_length=("Min words per example.", "option", "nw", int),
     seed=("Seed for random number generators", "option", "s", float),
     nr_iter=("Number of iterations to pretrain", "option", "i", int),
 )
@@ -42,6 +45,9 @@ def pretrain(
     use_vectors=False,
     dropout=0.2,
     nr_iter=1000,
+    batch_size=3000,
+    max_length=500,
+    min_length=5,
     seed=0,
 ):
     """
@@ -109,9 +115,14 @@ def pretrain(
     msg.row(("#", "# Words", "Total Loss", "Loss", "w/s"), **row_settings)
     for epoch in range(nr_iter):
         for batch in util.minibatch_by_words(
-            ((text, None) for text in texts), size=3000
+            ((text, None) for text in texts), size=batch_size
         ):
-            docs = make_docs(nlp, [text for (text, _) in batch])
+            docs = make_docs(
+                nlp,
+                [text for (text, _) in batch],
+                max_length=max_length,
+                min_length=min_length,
+            )
             loss = make_update(model, docs, optimizer, drop=dropout)
             progress = tracker.update(epoch, loss, docs)
             if progress:
@@ -152,7 +163,7 @@ def make_update(model, docs, optimizer, drop=0.0, objective="L2"):
     return float(loss)
 
 
-def make_docs(nlp, batch, min_length=1, max_length=500):
+def make_docs(nlp, batch, min_length, max_length):
     docs = []
     for record in batch:
         text = record["text"]
@@ -241,11 +252,23 @@ def update(self, epoch, loss, docs):
             status = (
                 epoch,
                 self.nr_word,
-                "%.8f" % self.loss,
-                "%.8f" % loss_per_word,
+                _smart_round(self.loss, width=10),
+                _smart_round(loss_per_word, width=6),
                 int(wps),
             )
             self.prev_loss = float(self.loss)
             return status
         else:
             return None
+
+
+def _smart_round(figure, width=10, max_decimal=4):
+    """Round large numbers as integers, smaller numbers as decimals."""
+    n_digits = len(str(int(figure)))
+    n_decimal = width - (n_digits + 1)
+    if n_decimal <= 1:
+        return str(int(figure))
+    else:
+        n_decimal = min(n_decimal, max_decimal)
+        format_str = "%." + str(n_decimal) + "f"
+        return format_str % figure


Test patch: