Repository: onnx/sklearn-onnx

Issue:
Multi label classifier (OneVsRestClassifier) normalization
Hi,

I am working on a multilabel pipeline (not multiclass): classifier may have multiple outputs.

## sklearn pipeline

I use OneVsRestClassifier on a LogisticRegression as a last step on a sklearn pipeline to get a multi-label classifier (not multiclass: my classifier may predict 0 classes, 1 class or several classes)

When I use predict_proba on my pipeline, I am able to get probabilities for each class, each probability of a class is independant from the one of another class.

```python
pipeline3 = Pipeline(steps=[('tfidf', TfidfVectorizer(input='content', analyzer='word',
                                                     min_df=5, max_df=0.5, ngram_range=(1, 1), lowercase=True)),
                           ('best', TruncatedSVD(n_components=50)),
                           ('lr', OneVsRestClassifier(LogisticRegression(penalty='l1',
                                                      solver='liblinear',
                                                      class_weight='balanced',
                                                      fit_intercept=True)))])
pipeline3.fit(X_train_preprocessed, y_train)
pipeline3.predict_proba(X_train_preprocessed.reshape(-1, 1)[0])
```
outputs array([[0.6818949 , 0.38450937, 0.39627797]]) 

- Class 0 is selected (>.5)
- Class 1 and 2 are not selected (< .5)

## Conversion to onnx

When I convert this pipeline with onnx, (convert_sklearn), these probabilities are normalized to sum to one (one vs rest classifier)

```python
from skl2onnx import convert_sklearn
from skl2onnx.common.data_types import StringTensorType

seps = {TfidfVectorizer: {"sep": [' ']}}
model3_onnx = convert_sklearn(pipeline3, name="tfidf",
                              initial_types=[("input", StringTensorType([1, 1]))],
                              options=seps)

with open("pipeline3_tfidf.onnx", "wb") as f:
    f.write(model3_onnx.SerializeToString())

# Inference session
import onnxruntime as rt

sess = rt.InferenceSession("pipeline3_tfidf.onnx")
inputs = {'input': X_train_preprocessed.reshape(-1, 1)[0]}
pred_onx = sess.run(None, inputs)
# print("predict", pred_onx[0])
print("predict_proba", pred_onx[1])
```
outputs [{0: 0.44900158047676086, 1: 0.2634965181350708, 2: 0.28750190138816833}]

I get a multiclass classifier (It corresponds to sklean output that was normalized afterwards).

I did not see any option within sklearn-onnx to bypass normalization step

## Calculation graph print and attempt to change outputs

Therefore I attempted to change the output of the calculation graph to get calculaitions before normalization step ('concatenated' step if I am right)

![model3_onnx dot](https://user-images.githubusercontent.com/31990506/56885801-a09af480-6a6d-11e9-9e55-f4aa68013be8.png)

```python
onnx_helper.select_model_inputs_outputs(model3_onnx, 'concatenated')

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-30-5926ab7e7f40> in <module>
----> 1 onnx_helper.select_model_inputs_outputs(model3_onnx, 'concatenated')

~/environment/default/lib/python3.7/site-packages/skl2onnx/helpers/onnx_helper.py in select_model_inputs_outputs(model, outputs, inputs)
    108             nb += 1
    109             for inp in node.input:
--> 110                 if mark_var[inp] == 1:
    111                     continue
    112                 mark_var[inp] = 1

KeyError: 'transform_matrix'
```

But it did not work out (Besides, if I cannot get a node to be an output as soon as I go after SVD step)

I suppose I am doing something wrong but I can't figure out what ?

Any help would be appreciated.

Regards,

Francois


Discussion:

Resolving patch:
diff --git a/skl2onnx/operator_converters/linear_regressor.py b/skl2onnx/operator_converters/linear_regressor.py
--- a/skl2onnx/operator_converters/linear_regressor.py
+++ b/skl2onnx/operator_converters/linear_regressor.py
@@ -12,10 +12,12 @@ def convert_sklearn_linear_regressor(scope, operator, container):
     op = operator.raw_operator
     op_type = 'LinearRegressor'
     attrs = {'name': scope.get_unique_operator_name(op_type)}
-    attrs['coefficients'] = op.coef_.astype(float)
+    attrs['coefficients'] = op.coef_.astype(float).ravel()
     attrs['intercepts'] = (op.intercept_.astype(float)
                            if isinstance(op.intercept_, collections.Iterable)
                            else [float(op.intercept_)])
+    if len(op.coef_.shape) == 2:
+        attrs['targets'] = op.coef_.shape[0]
     container.add_node(op_type, operator.input_full_names,
                        operator.output_full_names, op_domain='ai.onnx.ml',
                        **attrs)
diff --git a/skl2onnx/operator_converters/one_vs_rest_classifier.py b/skl2onnx/operator_converters/one_vs_rest_classifier.py
--- a/skl2onnx/operator_converters/one_vs_rest_classifier.py
+++ b/skl2onnx/operator_converters/one_vs_rest_classifier.py
@@ -3,11 +3,13 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-
+from sklearn.base import is_regressor
+from ..proto import onnx_proto
 from ..common._apply_operation import apply_concat
 from ..common._topology import FloatTensorType
 from ..common._registration import register_converter
-from ..common._apply_operation import apply_normalization, apply_slice
+from ..common._apply_operation import apply_normalization
+from ..common._apply_operation import apply_slice, apply_sub, apply_clip
 from ..common.utils_classifier import _finalize_converter_classes
 from .._supported_operators import sklearn_operator_name_map
 
@@ -25,36 +27,73 @@ def convert_one_vs_rest_classifier(scope, operator, container):
         this_operator.raw_operator = estimator
         this_operator.inputs = operator.inputs
 
-        label_name = scope.declare_local_variable('label_%d' % i)
-        prob_name = scope.declare_local_variable('proba_%d' % i,
-                                                 FloatTensorType())
-        this_operator.outputs.append(label_name)
-        this_operator.outputs.append(prob_name)
+        if is_regressor(estimator):
+            score_name = scope.declare_local_variable('score_%d' % i,
+                                                      FloatTensorType())
+            this_operator.outputs.append(score_name)
+
+            if len(estimator.coef_.shape) == 2:
+                raise RuntimeError("OneVsRestClassifier accepts "
+                                   "regressor with only one target.")
+            p1 = score_name.raw_name
+        else:
+            label_name = scope.declare_local_variable('label_%d' % i)
+            prob_name = scope.declare_local_variable('proba_%d' % i,
+                                                     FloatTensorType())
+            this_operator.outputs.append(label_name)
+            this_operator.outputs.append(prob_name)
 
-        # gets the probability for the class 1
-        p1 = scope.get_unique_variable_name('probY_%d' % i)
-        apply_slice(scope, prob_name.raw_name, p1, container, starts=[1],
-                    ends=[2], axes=[1],
-                    operator_name=scope.get_unique_operator_name('Slice'))
+            # gets the probability for the class 1
+            p1 = scope.get_unique_variable_name('probY_%d' % i)
+            apply_slice(scope, prob_name.raw_name, p1, container, starts=[1],
+                        ends=[2], axes=[1],
+                        operator_name=scope.get_unique_operator_name('Slice'))
 
         probs_names.append(p1)
 
-    # concatenates outputs
-    conc_name = scope.get_unique_variable_name('concatenated')
-    apply_concat(scope, probs_names, conc_name, container, axis=1)
+    if op.multilabel_:
+        # concatenates outputs
+        conc_name = operator.outputs[1].full_name
+        apply_concat(scope, probs_names, conc_name, container, axis=1)
+
+        # builds the labels (matrix with integer)
+        # scikit-learn may use probabilities or raw score
+        # but ONNX converters only uses probabilities.
+        # https://github.com/scikit-learn/scikit-learn/sklearn/
+        # multiclass.py#L290
+        # Raw score would mean: scores = conc_name.
+        thresh_name = scope.get_unique_variable_name('thresh')
+        container.add_initializer(
+            thresh_name, onnx_proto.TensorProto.FLOAT,
+            [1, len(op.classes_)], [.5] * len(op.classes_))
+        scores = scope.get_unique_variable_name('threshed')
+        apply_sub(scope, [conc_name, thresh_name], scores, container)
+
+        # sign
+        signed_input = scope.get_unique_variable_name('signed')
+        container.add_node('Sign', [scores], [signed_input],
+                           name=scope.get_unique_operator_name('Sign'))
+        # clip
+        apply_clip(scope, signed_input, operator.outputs[0].full_name,
+                   container, operator_name=None, max=None, min=0)
+    else:
+        # concatenates outputs
+        conc_name = scope.get_unique_variable_name('concatenated')
+        apply_concat(scope, probs_names, conc_name, container, axis=1)
 
-    # normalizes the outputs
-    apply_normalization(scope, conc_name, operator.outputs[1].full_name,
-                        container, axis=1, p=1)
+        # normalizes the outputs
+        apply_normalization(scope, conc_name, operator.outputs[1].full_name,
+                            container, axis=1, p=1)
 
-    # extracts the labels
-    label_name = scope.get_unique_variable_name('label_name')
-    container.add_node('ArgMax', conc_name, label_name,
-                       name=scope.get_unique_operator_name('ArgMax'), axis=1)
+        # extracts the labels
+        label_name = scope.get_unique_variable_name('label_name')
+        container.add_node('ArgMax', conc_name, label_name,
+                           name=scope.get_unique_operator_name('ArgMax'),
+                           axis=1)
 
-    _finalize_converter_classes(scope, label_name,
-                                operator.outputs[0].full_name, container,
-                                op.classes_)
+        _finalize_converter_classes(scope, label_name,
+                                    operator.outputs[0].full_name, container,
+                                    op.classes_)
 
 
 register_converter('SklearnOneVsRestClassifier',


Test patch:
diff --git a/tests/test_sklearn_glm_regressor_converter.py b/tests/test_sklearn_glm_regressor_converter.py
--- a/tests/test_sklearn_glm_regressor_converter.py
+++ b/tests/test_sklearn_glm_regressor_converter.py
@@ -17,6 +17,12 @@ def _fit_model(self, model):
         model.fit(X, y)
         return model, X
 
+    def _fit_model_multi(self, model):
+        X, y = datasets.make_regression(n_features=4, random_state=0,
+                                        n_targets=2, n_samples=10)
+        model.fit(X, y)
+        return model, X
+
     def test_model_linear_regression(self):
         model, X = self._fit_model(linear_model.LinearRegression())
         model_onnx = convert_sklearn(model, "linear regression",
@@ -108,6 +114,19 @@ def test_model_lasso_lars(self):
             basename="SklearnLassoLars-Dec4",
         )
 
+    def test_model_multi_linear_regression(self):
+        model, X = self._fit_model_multi(linear_model.LinearRegression())
+        model_onnx = convert_sklearn(model, "linear regression",
+                                     [("input", FloatTensorType([1, 4]))])
+        self.assertIsNotNone(model_onnx)
+        dump_data_and_model(
+            X.astype(numpy.float32),
+            model,
+            model_onnx,
+            verbose=False,
+            basename="SklearnLinearRegression-Dec4",
+        )
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_sklearn_one_vs_rest_classifier_converter.py b/tests/test_sklearn_one_vs_rest_classifier_converter.py
--- a/tests/test_sklearn_one_vs_rest_classifier_converter.py
+++ b/tests/test_sklearn_one_vs_rest_classifier_converter.py
@@ -1,14 +1,17 @@
 import unittest
-from sklearn.linear_model import LogisticRegression
+from sklearn.linear_model import LogisticRegression, LinearRegression
 from sklearn.multiclass import OneVsRestClassifier
 from skl2onnx.common.data_types import onnx_built_with_ml
-from test_utils import dump_multiple_classification
+from test_utils import (
+    dump_multiple_classification,
+    dump_multilabel_classification
+)
 
 
 class TestOneVsRestClassifierConverter(unittest.TestCase):
     @unittest.skipIf(not onnx_built_with_ml(),
                      reason="Requires ONNX-ML extension.")
-    def test_ova(self):
+    def test_ovr(self):
         model = OneVsRestClassifier(LogisticRegression())
         dump_multiple_classification(
             model,
@@ -18,7 +21,7 @@ def test_ova(self):
 
     @unittest.skipIf(not onnx_built_with_ml(),
                      reason="Requires ONNX-ML extension.")
-    def test_ova_02(self):
+    def test_ovr_02(self):
         model = OneVsRestClassifier(LogisticRegression())
         dump_multiple_classification(
             model,
@@ -30,7 +33,7 @@ def test_ova_02(self):
 
     @unittest.skipIf(not onnx_built_with_ml(),
                      reason="Requires ONNX-ML extension.")
-    def test_ova_string(self):
+    def test_ovr_string(self):
         model = OneVsRestClassifier(LogisticRegression())
         dump_multiple_classification(
             model,
@@ -41,6 +44,62 @@ def test_ova_string(self):
                           " <= StrictVersion('0.2.1')",
         )
 
+    @unittest.skipIf(not onnx_built_with_ml(),
+                     reason="Requires ONNX-ML extension.")
+    def test_ovr_multilabel(self):
+        model = OneVsRestClassifier(LogisticRegression())
+        dump_multilabel_classification(
+            model,
+            verbose=False,
+            suffix="MultiLabel",
+            allow_failure="StrictVersion(onnxruntime.__version__)"
+                          " <= StrictVersion('0.2.1')",
+        )
+        dump_multilabel_classification(
+            model,
+            verbose=False,
+            label_string=True,
+            suffix="MultiLabelString",
+            allow_failure="StrictVersion(onnxruntime.__version__)"
+                          " <= StrictVersion('0.2.1')",
+        )
+        dump_multilabel_classification(
+            model,
+            verbose=False,
+            first_class=2,
+            suffix="MultiLabel",
+            allow_failure="StrictVersion(onnxruntime.__version__)"
+                          " <= StrictVersion('0.2.1')",
+        )
+
+    @unittest.skipIf(not onnx_built_with_ml(),
+                     reason="Requires ONNX-ML extension.")
+    def test_ovr_multilabel_reg(self):
+        model = OneVsRestClassifier(LinearRegression())
+        dump_multilabel_classification(
+            model,
+            verbose=False,
+            suffix="MultiLabelReg-Out0",
+            allow_failure="StrictVersion(onnxruntime.__version__)"
+                          " <= StrictVersion('0.2.1')",
+        )
+        dump_multilabel_classification(
+            model,
+            verbose=False,
+            label_string=False,
+            suffix="MultiLabelRegString-Out0",
+            allow_failure="StrictVersion(onnxruntime.__version__)"
+                          " <= StrictVersion('0.2.1')",
+        )
+        dump_multilabel_classification(
+            model,
+            verbose=False,
+            first_class=2,
+            suffix="MultiLabelReg-Out0",
+            allow_failure="StrictVersion(onnxruntime.__version__)"
+                          " <= StrictVersion('0.2.1')",
+        )
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py
--- a/tests/test_utils/__init__.py
+++ b/tests/test_utils/__init__.py
@@ -9,6 +9,7 @@
 from .tests_helper import (  # noqa
     dump_one_class_classification,
     dump_binary_classification,
+    dump_multilabel_classification,
     dump_multiple_classification,
 )
 from .tests_helper import (  # noqa
diff --git a/tests/test_utils/tests_helper.py b/tests/test_utils/tests_helper.py
--- a/tests/test_utils/tests_helper.py
+++ b/tests/test_utils/tests_helper.py
@@ -12,8 +12,12 @@
 import platform
 import numpy
 import pandas
-from sklearn.datasets import make_classification
+from sklearn.datasets import (
+    make_classification,
+    make_multilabel_classification
+)
 from sklearn.base import BaseEstimator
+from sklearn.preprocessing import MultiLabelBinarizer
 from .utils_backend import (
     compare_backend,
     extract_options,
@@ -478,14 +482,81 @@ def dump_multiple_classification(
     )
 
 
+def dump_multilabel_classification(
+        model,
+        suffix="",
+        folder=None,
+        allow_failure=None,
+        verbose=False,
+        label_string=False,
+        first_class=0,
+        comparable_outputs=None):
+    """
+    Trains and dumps a model for a binary classification problem.
+    The function trains a model and calls
+    :func:`dump_data_and_model`.
+
+    Every created filename will follow the pattern:
+    ``<folder>/<prefix><task><classifier-name><suffix>.<data|expected|model|onnx>.<pkl|onnx>``.
+    """
+    X = [[0, 1], [1, 1], [2, 0], [0.5, 0.5], [1.1, 1.1], [2.1, 0.1]]
+    X = numpy.array(X, dtype=numpy.float32)
+    if label_string:
+        y = [["l0"], ["l1"], ["l2"], ["l0", "l1"], ["l1"], ["l2"]]
+    else:
+        y = [[0 + first_class], [1 + first_class], [2 + first_class],
+             [0 + first_class, 1 + first_class],
+             [1 + first_class], [2 + first_class]]
+    y = MultiLabelBinarizer().fit_transform(y)
+    model.fit(X, y)
+    if verbose:
+        print("[make_multilabel_classification] model '{}'".format(
+            model.__class__.__name__))
+    model_onnx, prefix = convert_model(model, "multi-class classifier",
+                                       [("input", FloatTensorType([1, 2]))])
+    if verbose:
+        print("[make_multilabel_classification] model was converted")
+    dump_data_and_model(
+        X.astype(numpy.float32),
+        model,
+        model_onnx,
+        folder=folder,
+        allow_failure=allow_failure,
+        basename=prefix + "Mcl" + model.__class__.__name__ + suffix,
+        verbose=verbose,
+        comparable_outputs=comparable_outputs,
+    )
+
+    X, y = make_multilabel_classification(40, n_features=4, random_state=42,
+                                          n_classes=3)
+    X = X[:, :2]
+    model.fit(X, y)
+    if verbose:
+        print("[make_multilabel_classification] model '{}'".format(
+            model.__class__.__name__))
+    model_onnx, prefix = convert_model(model, "multi-class classifier",
+                                       [("input", FloatTensorType([1, 2]))])
+    if verbose:
+        print("[make_multilabel_classification] model was converted")
+    dump_data_and_model(
+        X[:10].astype(numpy.float32),
+        model,
+        model_onnx,
+        folder=folder,
+        allow_failure=allow_failure,
+        basename=prefix + "RndMla" + model.__class__.__name__ + suffix,
+        verbose=verbose,
+        comparable_outputs=comparable_outputs,
+    )
+
+
 def dump_multiple_regression(
         model,
         suffix="",
         folder=None,
         allow_failure=None,
         comparable_outputs=None,
-        verbose=False,
-):
+        verbose=False):
     """
     Trains and dumps a model for a multi regression problem.
     The function trains a model and calls