Repository: onnx/sklearn-onnx

Issue:
MemoryError when trying to convert TfIdf
Hello
Got an exception when trying to export a pipeline with a TfIdf : 
Exception "unhandled MemoryError"
    cv = CountVectorizer()
    tt = TfidfTransformer()
    lsvc = LinearSVC(penalty=penalty, dual=False, tol=1e-3)
    text_clf = Pipeline([ 
        ('vect', cv),
        ('tfidf', tt),
        ('clf', lsvc),
    ])
    text_clf.fit(twenty_train.data, twenty_train.target) 
    print("Converting text_clf to onnx...")
    onnx = convert_sklearn(text_clf, target_opset=9, name='DocClassifierCV-Tfidf-LSVC', 
      initial_types=[('input', StringTensorType())]
      )

Exception "unhandled MemoryError"

The stack is:

convert_sklearn()

convert_topology() : 
            _registration.get_converter(operator.type)(scope, operator, container)

convert_sklearn_tfidf_transformer()
        if not isinstance(cst, numpy.ndarray):
            cst = numpy.array(cst.todense())

toDense()
     return np.asmatrix(self.toarray(order=order, out=out))

_process_toarray_args()
      return np.zeros(self.shape, dtype=self.dtype, order=order)

Could make sens : the input sparse matrix is (strangely) 130000 per 130000, pretty big to be densified.



Discussion:
https://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html#building-a-pipeline


Resolving patch:
diff --git a/skl2onnx/operator_converters/TfIdfTransformer.py b/skl2onnx/operator_converters/TfIdfTransformer.py
--- a/skl2onnx/operator_converters/TfIdfTransformer.py
+++ b/skl2onnx/operator_converters/TfIdfTransformer.py
@@ -4,9 +4,10 @@
 # license information.
 # --------------------------------------------------------------------------
 
-import numpy
 import numbers
 import warnings
+import numpy
+from scipy.sparse import diags
 from ..common._registration import register_converter
 from ..common._apply_operation import apply_log, apply_add, apply_mul, apply_identity
 from ..proto import onnx_proto
@@ -44,7 +45,11 @@ def convert_sklearn_tfidf_transformer(scope, operator, container):
         # X = X * self._idf_diag
         cst = op._idf_diag.astype(numpy.float32)
         if not isinstance(cst, numpy.ndarray):
-            cst = numpy.array(cst.todense())
+            if len(cst.shape) > 1:
+                n = cst.shape[0]
+                cst = numpy.array([cst[i, i] for i in range(n)])
+            else:
+                cst = numpy.array(cst.todense())
         if len(cst.shape) > 1:
             cst = numpy.diag(cst)
         cst = cst.ravel().flatten()


Test patch:
diff --git a/tests/test_SklearnPipeline.py b/tests/test_SklearnPipeline.py
--- a/tests/test_SklearnPipeline.py
+++ b/tests/test_SklearnPipeline.py
@@ -38,9 +38,6 @@ def transform(self, inp):
 class TestSklearnPipeline(unittest.TestCase):
 
     def test_pipeline(self):
-        from sklearn.preprocessing import StandardScaler
-        from sklearn.pipeline import Pipeline
-
         data = numpy.array([[0, 0], [0, 0], [1, 1], [1, 1]], dtype=numpy.float32)
         scaler = StandardScaler()
         scaler.fit(data)
@@ -51,9 +48,6 @@ def test_pipeline(self):
         dump_data_and_model(data, model, model_onnx, basename="SklearnPipelineScaler")
 
     def test_combine_inputs(self):
-        from sklearn.preprocessing import StandardScaler
-        from sklearn.pipeline import Pipeline
-
         data = numpy.array([[0., 0.], [0., 0.], [1., 1.], [1., 1.]], dtype=numpy.float32)
         scaler = StandardScaler()
         scaler.fit(data)
@@ -88,9 +82,6 @@ def test_combine_inputs_union_in_pipeline(self):
                             basename="SklearnPipelineScaler11Union-OneOff")
 
     def test_combine_inputs_floats_ints(self):
-        from sklearn.preprocessing import StandardScaler
-        from sklearn.pipeline import Pipeline
-
         data = [[0, 0.],[0, 0.],[1, 1.],[1, 1.]]
         scaler = StandardScaler()
         scaler.fit(data)
diff --git a/tests/test_SklearnTfidfTransformerConverter.py b/tests/test_SklearnTfidfTransformerConverter.py
--- a/tests/test_SklearnTfidfTransformerConverter.py
+++ b/tests/test_SklearnTfidfTransformerConverter.py
@@ -1,5 +1,6 @@
+# coding: utf-8
 """
-Tests scikit-learn's binarizer converter.
+Tests scikit-learn's TfidfTransformer converter.
 """
 import unittest
 import numpy
@@ -18,7 +19,8 @@ def test_model_tfidf_transform(self):
                 'This document is the second document.',
                 'And this is the third one.',
                 'Is this the first document?',
-                ]).reshape((4, 1))
+                "Troisième document en français",
+                ]).reshape((5, 1))
         data = CountVectorizer(ngram_range=(1, 1)).fit_transform(corpus.ravel()).todense()
         data = data.astype(numpy.float32)
         
diff --git a/tests/test_SklearnTfidfTransformerConverterSparse.py b/tests/test_SklearnTfidfTransformerConverterSparse.py
new file mode 100644
--- /dev/null
+++ b/tests/test_SklearnTfidfTransformerConverterSparse.py
@@ -0,0 +1,38 @@
+# coding: utf-8
+"""
+Tests examples from scikit-learn's documentation.
+"""
+from distutils.version import StrictVersion
+import unittest
+import numpy
+import onnx
+from sklearn.datasets import fetch_20newsgroups
+from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+from sklearn.pipeline import Pipeline
+from skl2onnx.common.data_types import StringTensorType
+from skl2onnx import convert_sklearn
+from test_utils import dump_data_and_model
+
+
+class TestSklearnTfidfVectorizerSparse(unittest.TestCase):
+
+    @unittest.skipIf(StrictVersion(onnx.__version__) <= StrictVersion('1.4.1'),
+                     # issue with encoding
+                     reason="https://github.com/onnx/onnx/pull/1734")
+    def test_model_tfidf_transform_bug(self):        
+        categories = ['alt.atheism', 'soc.religion.christian', 'comp.graphics', 'sci.med']
+        twenty_train = fetch_20newsgroups(subset='train', categories=categories,
+                                          shuffle=True, random_state=0)
+        text_clf = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer())])
+        twenty_train.data[0] = "bruît " + twenty_train.data[0]
+        text_clf.fit(twenty_train.data, twenty_train.target)
+        model_onnx = convert_sklearn(text_clf, name='DocClassifierCV-Tfidf',
+                                     initial_types=[('input', StringTensorType())])
+        dump_data_and_model(twenty_train.data[:10], text_clf, model_onnx,
+                            basename="SklearnPipelineTfidfTransformer",
+                            # Operator mul is not implemented in onnxruntime
+                            allow_failure="StrictVersion(onnx.__version__) <= StrictVersion('1.4.1')")
+
+
+if __name__ == "__main__":
+    unittest.main()