Issue 1:
sklearn.feature_extraction.text.HashingVectorizer uses sklearn/feature_extraction/_hashing.pyx which has a signed 32bit variable as a counter which overflows on my dataset and causes the following error:

Traceback (most recent call last): [0/1999]
File "/home/erg/foo/foo.py", line 109, in fit_transform
return self.foo.fit_transform(X)
File "/usr/lib/python3.6/site-packages/sklearn/pipeline.py", line 290, in fit_transform
Xt, fit_params = self._fit(X, y, **fit_params)
File "/usr/lib/python3.6/site-packages/sklearn/pipeline.py", line 222, in _fit
**fit_params_steps[name])
File "/usr/lib/python3.6/site-packages/sklearn/externals/joblib/memory.py", line 362, in __call__
return self.func(*args, **kwargs)
File "/usr/lib/python3.6/site-packages/sklearn/pipeline.py", line 589, in _fit_transform_one
res = transformer.fit_transform(X, y, **fit_params)
File "/usr/lib/python3.6/site-packages/sklearn/base.py", line 518, in fit_transform
return self.fit(X, **fit_params).transform(X)
File "/usr/lib/python3.6/site-packages/sklearn/feature_extraction/text.py", line 519, in transform
X = self._get_hasher().transform(analyzer(doc) for doc in X)
File "/usr/lib/python3.6/site-packages/sklearn/feature_extraction/hashing.py", line 167, in transform
shape=(n_samples, self.n_features))
File "/usr/lib/python3.6/site-packages/scipy/sparse/compressed.py", line 98, in __init__
self.check_format(full_check=False)
File "/usr/lib/python3.6/site-packages/scipy/sparse/compressed.py", line 167, in check_format
raise ValueError("indices and data should have the same size")
ValueError: indices and data should have the same size

Applying this patch and running my code gives an error:

diff --git a/sklearn/feature_extraction/_hashing.pyx b/sklearn/feature_extraction/_hashing.pyx
index e39aeafa0..b80d932cb 100644
--- a/sklearn/feature_extraction/_hashing.pyx
+++ b/sklearn/feature_extraction/_hashing.pyx
@@ -79,4 +79,6 @@ def transform(raw_X, Py_ssize_t n_features, dtype, bint alternate_sign=1):
indptr[len(indptr) - 1] = size

indices_a = np.frombuffer(indices, dtype=np.int32)
+ if not len(indices_a) == size:
+ raise ValueError("len indices_a: " + str(len(indices_a)) + ", size: " + str(size))

Traceback (most recent call last):
File "/usr/lib/python3.6/site-packages/sklearn/pipeline.py", line 289, in fit_transform
Xt, fit_params = self._fit(X, y, **fit_params)
File "/usr/lib/python3.6/site-packages/sklearn/pipeline.py", line 221, in _fit
**fit_params_steps[name])
File "/usr/lib/python3.6/site-packages/sklearn/externals/joblib/memory.py", line 362, in __call__
return self.func(*args, **kwargs)
File "/usr/lib/python3.6/site-packages/sklearn/pipeline.py", line 588, in _fit_transform_one
res = transformer.fit_transform(X, y, **fit_params)
File "/usr/lib/python3.6/site-packages/sklearn/base.py", line 518, in fit_transform
return self.fit(X, **fit_params).transform(X)
File "/usr/lib/python3.6/site-packages/sklearn/feature_extraction/text.py", line 519, in transform
X = self._get_hasher().transform(analyzer(doc) for doc in X)
File "/usr/lib/python3.6/site-packages/sklearn/feature_extraction/hashing.py", line 160, in transform
self.alternate_sign)
File "sklearn/feature_extraction/_hashing.pyx", line 83, in sklearn.feature_extraction._hashing.transform
ValueError: len indices_a: 2532660308, size: -1762306988

Proposed fix:

diff --git a/sklearn/feature_extraction/_hashing.pyx b/sklearn/feature_extraction/_hashing.pyx
index e39aeafa0..a3aec8158 100644
--- a/sklearn/feature_extraction/_hashing.pyx
+++ b/sklearn/feature_extraction/_hashing.pyx
@@ -38,7 +38,7 @@ def transform(raw_X, Py_ssize_t n_features, dtype, bint alternate_sign=1):
# Since Python array does not understand Numpy dtypes, we grow the indices
# and values arrays ourselves. Use a Py_ssize_t capacity for safety.
cdef Py_ssize_t capacity = 8192 # arbitrary
- cdef np.int32_t size = 0
+ cdef np.uint64_t size = 0
cdef np.ndarray values = np.empty(capacity, dtype=dtype)

for x in raw_X:

Issue 2:
Description

sklearn.feature_extraction.text.HashingVectorizer.fit_transform raises ValueError: indices and data should have the same size for data of a certain length. If you chunk the same data it runs fine.

Steps/Code to Reproduce

import sklearn
from sklearn.feature_extraction.text import HashingVectorizer
print('scikit-learn version')
print(sklearn.__version__)
vectorizer = HashingVectorizer(
analyzer='char', non_negative=True,
n_features=1024, ngram_range=[4,16])
X = ['A'*1432]*203452
print('works')
vectorizer.fit_transform(X[:100000])
print('does not work')
vectorizer.fit_transform(X)

Expected Results

scikit-learn version
0.18.1
works
does not work

Actual Results

scikit-learn version
0.18.1
works
does not work
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-1-aae200adab09> in <module>()
10 vectorizer.fit_transform(X[:100000])
11 print('does not work')
---> 12 vectorizer.fit_transform(X)

/Users/benkaehler/miniconda3/envs/qiime2-2017.4/lib/python3.5/site-packages/sklearn/feature_extraction/text.py in transform(self, X, y)
485
486 analyzer = self.build_analyzer()
--> 487 X = self._get_hasher().transform(analyzer(doc) for doc in X)
488 if self.binary:
489 X.data.fill(1)

/Users/benkaehler/miniconda3/envs/qiime2-2017.4/lib/python3.5/site-packages/sklearn/feature_extraction/hashing.py in transform(self, raw_X, y)
147
148 X = sp.csr_matrix((values, indices, indptr), dtype=self.dtype,
--> 149 shape=(n_samples, self.n_features))
150 X.sum_duplicates() # also sorts the indices
151 if self.non_negative:

/Users/benkaehler/miniconda3/envs/qiime2-2017.4/lib/python3.5/site-packages/scipy/sparse/compressed.py in __init__(self, arg1, shape, dtype, copy)
96 self.data = np.asarray(self.data, dtype=dtype)
97
---> 98 self.check_format(full_check=False)
99
100 def getnnz(self, axis=None):

/Users/benkaehler/miniconda3/envs/qiime2-2017.4/lib/python3.5/site-packages/scipy/sparse/compressed.py in check_format(self, full_check)
165 # check index and data arrays
166 if (len(self.indices) != len(self.data)):
--> 167 raise ValueError("indices and data should have the same size")
168 if (self.indptr[-1] > len(self.indices)):
169 raise ValueError("Last value of index pointer should be less than "

ValueError: indices and data should have the same size

Versions

Darwin-16.5.0-x86_64-i386-64bit
Python 3.5.3 |Continuum Analytics, Inc.| (default, Mar 6 2017, 12:15:08)
[GCC 4.2.1 Compatible Apple LLVM 6.0 (clang-600.0.57)]
NumPy 1.12.1
SciPy 0.19.0
Scikit-Learn 0.18.1

and

Linux-2.6.32-504.16.2.el6.x86_64-x86_64-with-centos-6.6-Final
Python 3.5.3 |Continuum Analytics, Inc.| (default, Mar 6 2017, 11:58:13)
[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]
NumPy 1.12.1
SciPy 0.19.0
Scikit-Learn 0.18.1