Repository: onnx/sklearn-onnx

Issue:
NotImplementedError: metric='cosine' is not implemented.
I'm trying to use ONNX format as an universal container for models transfer, and I've found that I'm not able to export a simple kNN model with cosine distance, which is very common in NLP:

```python
clf = KNeighborsClassifier(7,weights='distance',metric='cosine',n_jobs=-1)
clf.fit(vectors_train, y_train)
...
initial_type = [('float_input', FloatTensorType([None, 4]))]
onx = convert_sklearn(knn, initial_types=initial_type)
```

gives
```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/proj/python/onnx/lib/python3.6/site-packages/skl2onnx/convert.py", line 162, in convert_sklearn
    remove_identity=not intermediate)
  File "/proj/python/onnx/lib/python3.6/site-packages/skl2onnx/common/_topology.py", line 1087, in convert_topology
    conv(scope, operator, container)
  File "/proj/python/onnx/lib/python3.6/site-packages/skl2onnx/common/_registration.py", line 29, in __call__
    return self._fct(*args)
  File "/proj/python/onnx/lib/python3.6/site-packages/skl2onnx/operator_converters/nearest_neighbours.py", line 442, in convert_nearest_neighbors_classifier
    many = _convert_nearest_neighbors(operator, container)
  File "/proj/python/onnx/lib/python3.6/site-packages/skl2onnx/operator_converters/nearest_neighbours.py", line 268, in _convert_nearest_neighbors
    **distance_kwargs)
  File "/proj/python/onnx/lib/python3.6/site-packages/skl2onnx/operator_converters/nearest_neighbours.py", line 92, in onnx_nearest_neighbors_indices_k
    **kwargs)
  File "/proj/python/onnx/lib/python3.6/site-packages/skl2onnx/algebra/complex_functions.py", line 105, in onnx_cdist
    metric))
NotImplementedError: metric='cosine' is not implemented.

```
which is true in current master: https://github.com/onnx/sklearn-onnx/blob/a8267e7ba946d8b0596951060e5dca39fec47439/skl2onnx/algebra/complex_functions.py#L83-L105

And actually searching through the repository, there's no single mention of `cosine` in the whole code base.

So, I'd like to clarify, is there any point in waiting for it to be implemented in not-too-distant future, or it's more reasonable to abandon hope and move somewhere else?


Discussion:

Resolving patch:
diff --git a/skl2onnx/algebra/onnx_ops.py b/skl2onnx/algebra/onnx_ops.py
--- a/skl2onnx/algebra/onnx_ops.py
+++ b/skl2onnx/algebra/onnx_ops.py
@@ -271,6 +271,22 @@ def OnnxUnsqueezeApi11(*x, axes=None, op_version=None,
                            op_version=op_version, output_names=output_names)
 
 
+def OnnxReduceL2_typed(dtype, x, axes=None, keepdims=1, op_version=None,
+                       output_names=None):
+    """
+    Adds operator ReduceL2 for float or double.
+    """
+    if dtype == np.float32:
+        return OnnxReduceL2(  # noqa
+            x, axes=axes, keepdims=keepdims,
+            op_version=op_version, output_names=output_names)
+    x2 = OnnxMul(x, x, op_version=op_version)  # noqa
+    red = OnnxReduceSumApi11(
+        x2, axes=[1], keepdims=1, op_version=op_version)
+    return OnnxSqrt(  # noqa
+        red, op_version=op_version, output_names=output_names)
+
+
 def OnnxReshapeApi13(*x, allowzero=0, op_version=None,
                      output_names=None):
     """
diff --git a/skl2onnx/operator_converters/_gp_kernels.py b/skl2onnx/operator_converters/_gp_kernels.py
--- a/skl2onnx/operator_converters/_gp_kernels.py
+++ b/skl2onnx/operator_converters/_gp_kernels.py
@@ -5,7 +5,7 @@
 from sklearn.gaussian_process.kernels import (
     Sum, Product, ConstantKernel,
     RBF, DotProduct, ExpSineSquared,
-    RationalQuadratic
+    RationalQuadratic, PairwiseKernel
 )
 from ..algebra.complex_functions import onnx_squareform_pdist, onnx_cdist
 from ..algebra.onnx_ops import (
@@ -13,7 +13,8 @@
     OnnxTranspose, OnnxDiv, OnnxExp,
     OnnxShape, OnnxSin, OnnxPow,
     OnnxReduceSumApi11, OnnxSqueezeApi11,
-    OnnxIdentity, OnnxReduceSumSquare
+    OnnxIdentity, OnnxReduceSumSquare,
+    OnnxReduceL2_typed
 )
 from ..algebra.custom_ops import OnnxCDist
 from ..proto.onnx_helper_modified import from_array
@@ -90,6 +91,9 @@ def py_make_float_array(cst, dtype, as_tensor=False):
 def _convert_exp_sine_squared(X, Y, length_scale=1.2, periodicity=1.1,
                               pi=math.pi, dtype=None, optim=None,
                               op_version=None, **kwargs):
+    """
+    Implements the kernel ExpSineSquared.
+    """
     if optim is None:
         dists = onnx_cdist(
             X, Y, metric="euclidean", dtype=dtype, op_version=op_version)
@@ -118,7 +122,7 @@ def _convert_exp_sine_squared(X, Y, length_scale=1.2, periodicity=1.1,
 def _convert_dot_product(X, Y, sigma_0=2.0, dtype=None, op_version=None,
                          **kwargs):
     """
-    Implements the kernel
+    Implements the kernel DotProduct.
     :math:`k(x_i,x_j)=\\sigma_0^2+x_i\\cdot x_j`.
     """
     # It only works in two dimensions.
@@ -136,7 +140,7 @@ def _convert_rational_quadratic(X, Y, length_scale=1.0, alpha=2.0,
                                 dtype=None, optim=None, op_version=None,
                                 **kwargs):
     """
-    Implements the kernel
+    Implements the kernel RationalQuadratic.
     :math:`k(x_i,x_j)=(1 + d(x_i, x_j)^2 / (2*\\alpha * l^2))^{-\\alpha}`.
     """
     if optim is None:
@@ -156,6 +160,40 @@ def _convert_rational_quadratic(X, Y, length_scale=1.0, alpha=2.0,
     return OnnxIdentity(K, op_version=op_version, **kwargs)
 
 
+def _convert_pairwise_kernel(X, Y, metric=None,
+                             dtype=None, op_version=None,
+                             gamma=None, gamma_bounds=None,
+                             degree=None, coef0=None,
+                             optim=None, **kwargs):
+    """
+    Implements the kernel PairwiseKernel.
+
+    * cosine: :math:`k(x_i,x_j)=\\frac{<x_i, x_j>}
+        {\\parallel x_i \\parallel \\parallel x_j \\parallel}`
+
+    See `KERNEL_PARAMS
+    <https://github.com/scikit-learn/scikit-learn/blob/
+    95119c13af77c76e150b753485c662b7c52a41a2/sklearn/
+    metrics/pairwise.py#L1862>`_.
+    """
+    if metric == 'cosine':
+        if isinstance(Y, np.ndarray):
+            ny = np.sqrt(np.sum(Y ** 2, axis=1, keepdims=True))
+            norm_y = Y / ny
+            norm_try = norm_y.T.astype(dtype)
+        else:
+            ny = OnnxReduceL2_typed(dtype, Y, axes=[1], op_version=op_version)
+            norm_y = OnnxDiv(Y, ny, op_version=op_version)
+            norm_try = OnnxTranspose(norm_y, perm=[1, 0],
+                                     op_version=op_version)
+
+        nx = OnnxReduceL2_typed(dtype, X, axes=[1], op_version=op_version)
+        norm_x = OnnxDiv(X, nx, op_version=op_version)
+        K = OnnxMatMul(norm_x, norm_try, op_version=op_version)
+        return OnnxIdentity(K, op_version=op_version, **kwargs)
+    raise NotImplementedError("Metric %r is not implemented." % metric)
+
+
 def convert_kernel(kernel, X, output_names=None,
                    x_train=None, dtype=None, optim=None,
                    op_version=None):
@@ -291,6 +329,18 @@ def convert_kernel(kernel, X, output_names=None,
                 output_names=output_names,
                 optim=optim, op_version=op_version)
 
+    if isinstance(kernel, PairwiseKernel):
+        if x_train is None:
+            return _convert_pairwise_kernel(
+                X, X, metric=kernel.metric,
+                dtype=dtype, output_names=output_names,
+                optim=optim, op_version=op_version)
+        else:
+            return _convert_pairwise_kernel(
+                X, x_train, metric=kernel.metric,
+                dtype=dtype, output_names=output_names,
+                optim=optim, op_version=op_version)
+
     raise RuntimeError("Unable to convert __call__ method for "
                        "class {}.".format(type(kernel)))
 


Test patch:
diff --git a/tests/test_sklearn_gaussian_process_regressor.py b/tests/test_sklearn_gaussian_process_regressor.py
--- a/tests/test_sklearn_gaussian_process_regressor.py
+++ b/tests/test_sklearn_gaussian_process_regressor.py
@@ -13,7 +13,7 @@
 from sklearn.gaussian_process import GaussianProcessRegressor
 from sklearn.gaussian_process.kernels import (
     Sum, DotProduct, ExpSineSquared, RationalQuadratic,
-    RBF, ConstantKernel as C,
+    RBF, ConstantKernel as C, PairwiseKernel
 )
 from sklearn.model_selection import train_test_split
 from skl2onnx.common.data_types import FloatTensorType, DoubleTensorType
@@ -27,6 +27,10 @@
     from onnxruntime import GraphOptimizationLevel
 except ImportError:
     GraphOptimizationLevel = None
+try:
+    from onnxruntime.capi.onnxruntime_pybind11_state import NotImplemented
+except ImportError:
+    NotImplemented = RuntimeError
 from onnxruntime import __version__ as ort_version
 from test_utils import dump_data_and_model, fit_regression_model, TARGET_OPSET
 
@@ -152,6 +156,68 @@ def test_kernel_constant1(self):
         m2 = ker(Xtest_)
         assert_almost_equal(m1, m2, decimal=5)
 
+    @unittest.skipIf(
+        StrictVersion(ort_version) <= StrictVersion(THRESHOLD),
+        reason="onnxruntime %s" % THRESHOLD)
+    def test_kernel_cosine_float(self):
+        ker = PairwiseKernel(metric='cosine')
+
+        # X, X
+        onx = convert_kernel(ker, 'X', output_names=['Y'], dtype=np.float32,
+                             op_version=_TARGET_OPSET_)
+        model_onnx = onx.to_onnx(
+            inputs=[('X', FloatTensorType([None, None]))],
+            target_opset=TARGET_OPSET)
+
+        x = np.random.randn(4, 3)
+        x[0, 0] = x[1, 1] = x[2, 2] = 10.
+        x[3, 2] = 5.
+
+        sess = InferenceSession(model_onnx.SerializeToString())
+        res = sess.run(None, {'X': x.astype(np.float32)})[0]
+        m1 = res
+        m2 = ker(x)
+        assert_almost_equal(m1, m2, decimal=5)
+
+        # X, x
+        onx = convert_kernel(ker, 'X', x_train=x,
+                             output_names=['Y'], dtype=np.float32,
+                             op_version=_TARGET_OPSET_)
+        model_onnx = onx.to_onnx(
+            inputs=[('X', FloatTensorType([None, None]))],
+            target_opset=TARGET_OPSET)
+
+        sess = InferenceSession(model_onnx.SerializeToString())
+        res = sess.run(None, {'X': x.astype(np.float32)})[0]
+        m1 = res
+        m2 = ker(x)
+        assert_almost_equal(m1, m2, decimal=5)
+
+    @unittest.skipIf(
+        StrictVersion(ort_version) <= StrictVersion(THRESHOLD),
+        reason="onnxruntime %s" % THRESHOLD)
+    def test_kernel_cosine_double(self):
+        ker = PairwiseKernel(metric='cosine')
+        onx = convert_kernel(ker, 'X', output_names=['Y'], dtype=np.float64,
+                             op_version=_TARGET_OPSET_)
+        model_onnx = onx.to_onnx(
+            inputs=[('X', DoubleTensorType([None, None]))],
+            target_opset=TARGET_OPSET)
+
+        x = np.random.randn(4, 3)
+        x[0, 0] = x[1, 1] = x[2, 2] = 10.
+        x[3, 2] = 5.
+
+        try:
+            sess = InferenceSession(model_onnx.SerializeToString())
+        except NotImplemented:
+            # Failed to find kernel for FusedMatMul(1).
+            return
+        res = sess.run(None, {'X': x.astype(np.float64)})[0]
+        m1 = res
+        m2 = ker(x)
+        assert_almost_equal(m1, m2, decimal=5)
+
     @unittest.skipIf(
         StrictVersion(ort_version) <= StrictVersion(THRESHOLD),
         reason="onnxruntime %s" % THRESHOLD)
@@ -502,6 +568,52 @@ def test_gpr_rbf_fitted_true(self):
                             verbose=False,
                             basename="SklearnGaussianProcessRBFTDouble")
 
+    @unittest.skipIf(
+        StrictVersion(ort_version) < StrictVersion("1.6.0"),
+        reason="shape_inference fails")
+    @unittest.skipIf(
+        StrictVersion(ort_version) <= StrictVersion(THRESHOLD),
+        reason="onnxruntime %s" % THRESHOLD)
+    def test_gpr_cosine_fitted_true_float(self):
+        gp = GaussianProcessRegressor(alpha=1e-5,
+                                      n_restarts_optimizer=25,
+                                      normalize_y=False,
+                                      kernel=PairwiseKernel(metric='cosine'))
+        gp, X = fit_regression_model(
+            gp, n_features=2, n_samples=20, factor=0.01)
+
+        # return_cov=False, return_std=False
+        model_onnx = to_onnx(
+            gp, initial_types=[('X', FloatTensorType([None, None]))],
+            target_opset=TARGET_OPSET)
+        self.assertTrue(model_onnx is not None)
+        dump_data_and_model(X.astype(np.float32), gp, model_onnx,
+                            verbose=False,
+                            basename="SklearnGaussianProcessCosineFloat-Dec2")
+
+    @unittest.skipIf(
+        StrictVersion(ort_version) < StrictVersion("1.6.0"),
+        reason="shape_inference fails")
+    @unittest.skipIf(
+        StrictVersion(ort_version) <= StrictVersion(THRESHOLD),
+        reason="onnxruntime %s" % THRESHOLD)
+    def test_gpr_cosine_fitted_true_double(self):
+        gp = GaussianProcessRegressor(alpha=1e-5,
+                                      n_restarts_optimizer=25,
+                                      normalize_y=False,
+                                      kernel=PairwiseKernel(metric='cosine'))
+        gp, X = fit_regression_model(
+            gp, n_features=2, n_samples=20, factor=0.01)
+
+        # return_cov=False, return_std=False
+        model_onnx = to_onnx(
+            gp, initial_types=[('X', DoubleTensorType([None, None]))],
+            target_opset=TARGET_OPSET)
+        self.assertTrue(model_onnx is not None)
+        dump_data_and_model(X.astype(np.float64), gp, model_onnx,
+                            verbose=False,
+                            basename="SklearnGaussianProcessCosineDouble")
+
     @unittest.skipIf(
         StrictVersion(ort_version) <= StrictVersion(THRESHOLD),
         reason="onnxruntime %s" % THRESHOLD)
@@ -562,6 +674,7 @@ def test_gpr_rbf_fitted_return_std_exp_sine_squared_true(self):
         X = 15 * state.rand(100, 2)
         y = np.sin(X[:, 0] - X[:, 1]).ravel()
         y += 0.5 * (0.5 - state.rand(X.shape[0]))
+        y /= 10
         X_train, X_test, y_train, _ = train_test_split(X, y)
         gp = GaussianProcessRegressor(
             kernel=ExpSineSquared(periodicity_bounds=(1e-10, 1e10)),