Repository: onnx/sklearn-onnx

Issue:
TfidfVectoriser converter issues
1) NaN values

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.datasets import fetch_20newsgroups

data = fetch_20newsgroups()
X, y = np.array(data.data)[:100], np.array(data.target)[:100]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)

model = TfidfVectorizer().fit(X_train)
onnx_model = convert_sklearn(model, 'cv', [('input', StringTensorType(X_test.shape))])
save_model(onnx_model, 'cv.onnx')
sess = InferenceSession('cv.onnx')
res = sess.run(None, input_feed={'input': X_test})

print(np.mean(np.isclose(model.transform(X_test).toarray(), res[0], atol=1e-5)))

0.971024886272411 / 0.05691196146641691 (Number of NaN outputs vary a lot in different runs)

model.transform(X_test).toarray()

array([[0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.0390878 , 0.        , ..., 0.        , 0.02345268,
        0.        ],
       ...,
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.0268699 , 0.        , ..., 0.        , 0.        ,
        0.        ]])

res[0]

array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],
       [nan, nan, nan, ..., nan, nan, nan],
       [ 0.,  0.,  0., ...,  0.,  0.,  0.],
       ...,
       [nan, nan, nan, ..., nan, nan, nan],
       [nan, nan, nan, ..., nan, nan, nan],
       [nan, nan, nan, ..., nan, nan, nan]], dtype=float32)

2) lowercase=False

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.datasets import fetch_20newsgroups

data = fetch_20newsgroups()
X, y = np.array(data.data)[:100], np.array(data.target)[:100]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)

model = TfidfVectorizer(lowercase=False).fit(X_train)
onnx_model = convert_sklearn(model, 'cv', [('input', StringTensorType(X_test.shape))])
save_model(onnx_model, 'cv.onnx')
sess = InferenceSession('cv.onnx')
res = sess.run(None, input_feed={'input': X_test})

print(np.mean(np.isclose(model.transform(X_test).toarray(), res[0], atol=1e-5)))

0.19266698024459078

model.transform(X_test).toarray()

array([[0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.04067757, 0.        , ..., 0.        , 0.        ,
        0.02440654],
       ...,
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.03138701, 0.        , ..., 0.        , 0.        ,
        0.        ]])


res[0]

array([[0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.03395877, 0.        , ..., 0.        , 0.        ,
        0.        ],
       ...,
       [       nan,        nan,        nan, ...,        nan,        nan,
               nan],
       [       nan,        nan,        nan, ...,        nan,        nan,
               nan],
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ]], dtype=float32)


3) analyzer='char'

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.datasets import fetch_20newsgroups

data = fetch_20newsgroups()
X, y = np.array(data.data)[:100], np.array(data.target)[:100]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)

model = TfidfVectorizer(analyzer='char').fit(X_train)
onnx_model = convert_sklearn(model, 'cv', [('input', StringTensorType(X_test.shape))])
save_model(onnx_model, 'cv.onnx')
sess = InferenceSession('cv.onnx')
res = sess.run(None, input_feed={'input': X_test})

print(np.mean(np.isclose(model.transform(X_test).toarray(), res[0], atol=1e-5)))

0.2504225352112676

model.transform(X_test).toarray()

array([[0.        , 0.04540119, 0.62653648, ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.07204533, 0.57293189, ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.01663145, 0.61892768, ..., 0.        , 0.        ,
        0.        ],
       ...,
       [0.        , 0.03520802, 0.46474592, ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.03261331, 0.34787535, ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.02589324, 0.61991454, ..., 0.        , 0.00645606,
        0.        ]])

res[0]

array([[0.16190134, 0.10120933, 0.74220175, ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [0.        , 0.        ,        nan, ..., 0.        , 0.        ,
        0.        ],
       ...,
       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,
        0.        ],
       [       nan,        nan,        nan, ...,        nan,        nan,
               nan],
       [0.        ,        nan,        nan, ..., 0.        , 0.        ,
        0.        ]], dtype=float32)



Discussion:
https://github.com/onnx/sklearn-onnx/pull/243

Resolving patch:
diff --git a/skl2onnx/operator_converters/text_vectoriser.py b/skl2onnx/operator_converters/text_vectoriser.py
--- a/skl2onnx/operator_converters/text_vectoriser.py
+++ b/skl2onnx/operator_converters/text_vectoriser.py
@@ -146,9 +146,16 @@ def convert_sklearn_text_vectorizer(scope, operator, container):
     if op.analyzer == "char_wb":
         raise NotImplementedError(
             "CountVectorizer cannot be converted, "
-            "only tokenizer='word' is supported. "
+            "only tokenizer='word' is fully supported. "
             "You may raise an issue at "
             "https://github.com/onnx/sklearn-onnx/issues.")
+    if op.analyzer == "char":
+        warnings.warn(
+            "The conversion of CountVectorizer may not work. "
+            "only tokenizer='word' is fully supported. "
+            "You may raise an issue at "
+            "https://github.com/onnx/sklearn-onnx/issues.",
+            UserWarning)
     if op.strip_accents is not None:
         raise NotImplementedError(
             "CountVectorizer cannot be converted, "


Test patch:
diff --git a/tests/test_sklearn_tfidf_vectorizer_converter_char.py b/tests/test_sklearn_tfidf_vectorizer_converter_char.py
--- a/tests/test_sklearn_tfidf_vectorizer_converter_char.py
+++ b/tests/test_sklearn_tfidf_vectorizer_converter_char.py
@@ -75,6 +75,29 @@ def test_model_tfidf_vectorizer11_char(self):
                           "StrictVersion('0.3.0')",
             verbose=False)
 
+    @unittest.skipIf(
+        StrictVersion(onnx.__version__) <= StrictVersion("1.4.1"),
+        reason="Requires opset 9.")
+    @unittest.skipIf(True, reason="expected failure")
+    def test_model_tfidf_vectorizer11_char_doublespace(self):
+        corpus = numpy.array([
+                'This is the first  document.',
+                'This document is the second document.',
+                ]).reshape((2, 1))
+        vect = TfidfVectorizer(ngram_range=(1, 1), norm=None,
+                               analyzer='char')
+        vect.fit(corpus.ravel())
+        model_onnx = convert_sklearn(vect, 'TfidfVectorizer',
+                                     [('input', StringTensorType([1]))])
+        self.assertTrue(model_onnx is not None)
+
+        dump_data_and_model(
+            corpus, vect, model_onnx,
+            basename="SklearnTfidfVectorizer11CharSpace-OneOff-SklCol",
+            allow_failure="StrictVersion(onnxruntime.__version__) <= "
+                          "StrictVersion('0.3.0')",
+            verbose=False)
+
     @unittest.skipIf(
         StrictVersion(onnx.__version__) <= StrictVersion("1.4.1"),
         reason="Requires opset 9.")
diff --git a/tests/test_sklearn_tfidf_vectorizer_converter_dataset.py b/tests/test_sklearn_tfidf_vectorizer_converter_dataset.py
new file mode 100644
--- /dev/null
+++ b/tests/test_sklearn_tfidf_vectorizer_converter_dataset.py
@@ -0,0 +1,53 @@
+"""
+Tests scikit-learn's tfidf converter using downloaded data.
+"""
+import unittest
+from distutils.version import StrictVersion
+import numpy as np
+import onnx
+from sklearn.model_selection import train_test_split
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.datasets import fetch_20newsgroups
+from skl2onnx import convert_sklearn
+from skl2onnx.common.data_types import StringTensorType
+from test_utils import dump_data_and_model
+
+
+class TestSklearnTfidfVectorizerDataSet(unittest.TestCase):
+
+    @unittest.skipIf(
+        StrictVersion(onnx.__version__) <= StrictVersion("1.4.1"),
+        reason="Requires opset 9.")
+    def test_tfidf_20newsgroups(self):
+        data = fetch_20newsgroups()
+        X, y = np.array(data.data)[:100], np.array(data.target)[:100]
+        X_train, X_test, y_train, y_test = train_test_split(
+            X, y, test_size=0.5, random_state=42)
+
+        model = TfidfVectorizer().fit(X_train)
+        onnx_model = convert_sklearn(
+            model, 'cv', [('input', StringTensorType(X_test.shape))])
+        dump_data_and_model(
+            X_test, model, onnx_model,
+            basename="SklearnTfidfVectorizer20newsgroups",
+            allow_failure="StrictVersion(onnxruntime.__version__)"
+                          " <= StrictVersion('0.4.0')")
+
+    def test_tfidf_20newsgroups_nolowercase(self):
+        data = fetch_20newsgroups()
+        X, y = np.array(data.data)[:100], np.array(data.target)[:100]
+        X_train, X_test, y_train, y_test = train_test_split(
+            X, y, test_size=0.5, random_state=42)
+
+        model = TfidfVectorizer(lowercase=False).fit(X_train)
+        onnx_model = convert_sklearn(
+            model, 'cv', [('input', StringTensorType(X_test.shape))])
+        dump_data_and_model(
+            X_test, model, onnx_model,
+            basename="SklearnTfidfVectorizer20newsgroupsNOLower",
+            allow_failure="StrictVersion(onnxruntime.__version__)"
+                          " <= StrictVersion('0.4.0')")
+
+
+if __name__ == "__main__":
+    unittest.main()