internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: microsoft/onnxscript

Issue:
Windows CI has MemoryError
First seen: https://github.com/microsoft/onnx-script/actions/runs/4641478064/jobs/8214491813,
Also https://github.com/microsoft/onnx-script/actions/runs/4642961358/jobs/8217281756


Discussion:
I wonder if the cache for the eager ort evaluator needs to be purged
Windows is now consistently running out of memory https://github.com/microsoft/onnx-script/actions/runs/4726705836/jobs/8386557760?pr=643

Resolving patch:
diff --git a/onnxscript/_internal/feature_switch.py b/onnxscript/_internal/feature_switch.py
new file mode 100644
--- /dev/null
+++ b/onnxscript/_internal/feature_switch.py
@@ -0,0 +1,10 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+"""Switches to determine if the corresponding feature of onnxscript is enabled or not."""
+
+import os
+
+# By default: Enable
+CACHE_ORT_SESSIONS: bool = os.getenv("CACHE_ORT_SESSIONS", "1") != "0"
diff --git a/onnxscript/evaluator.py b/onnxscript/evaluator.py
--- a/onnxscript/evaluator.py
+++ b/onnxscript/evaluator.py
@@ -27,7 +27,7 @@
 from typing_extensions import TypeAlias
 
 from onnxscript import autocast, irbuilder, onnx_opset, tensor, utils, values
-from onnxscript._internal import param_manipulation
+from onnxscript._internal import feature_switch, param_manipulation
 
 if typing.TYPE_CHECKING:
     import onnxruntime as ort
@@ -354,12 +354,16 @@ def _cache_(model, providers):
     import onnxruntime as ort  # pylint: disable=import-outside-toplevel
 
     serialized = model.SerializeToString()
-    key = serialized, tuple(providers)
-    if key in _cache_models:
-        return _cache_models[key]
-    session = ort.InferenceSession(serialized, providers=providers)
-    _cache_models[key] = session
-    return session
+    if feature_switch.CACHE_ORT_SESSIONS:
+        key = serialized, tuple(providers)
+        if key in _cache_models:
+            return _cache_models[key]
+        session = ort.InferenceSession(serialized, providers=providers)
+        _cache_models[key] = session
+
+        return session
+
+    return ort.InferenceSession(serialized, providers=providers)
 
 
 def _os_to_ort_value(v):


Test patch:
Repository: microsoft/onnxscript

Issue:
The latest torch exporter causes circular import in tests
https://github.com/microsoft/onnxscript/actions/runs/5929694882/job/16077910883?pr=1015




Discussion:

Resolving patch:
diff --git a/onnxscript/function_libs/torch_lib/graph_building.py b/onnxscript/function_libs/torch_lib/graph_building.py
--- a/onnxscript/function_libs/torch_lib/graph_building.py
+++ b/onnxscript/function_libs/torch_lib/graph_building.py
@@ -15,7 +15,6 @@
 import onnx.helper
 import onnx.shape_inference
 import torch
-from torch.onnx import _type_utils
 from typing_extensions import TypeAlias
 
 import onnxscript
@@ -157,6 +156,9 @@ def dtype(self) -> torch.dtype | None:
         # TODO: Return numpy dtype
         if self._torch_dtype is not None:
             return self._torch_dtype
+        # Local import to avoid circular dependency
+        from torch.onnx import _type_utils  # pylint: disable=import-outside-toplevel
+
         torch_dtype = _type_utils.JitScalarType.from_value(  # type: ignore[attr-defined]
             self._torch_value, default=_type_utils.JitScalarType.UNDEFINED
         )
@@ -180,6 +182,9 @@ def is_complex(self, is_complex: bool):
 
     @property
     def onnx_dtype(self):
+        # Local import to avoid circular dependency
+        from torch.onnx import _type_utils  # pylint: disable=import-outside-toplevel
+
         return _type_utils.JitScalarType.from_value(  # type: ignore[attr-defined]
             self._torch_value, _type_utils.JitScalarType.UNDEFINED
         ).onnx_type()


Test patch:
Repository: microsoft/onnxscript

Issue:
Use the correct logger in values.py
We should get the module logger first in

https://github.com/microsoft/onnxscript/blob/036b6478c7cdef0b5fdee4f564eef9c8d450002c/onnxscript/values.py#L287

Otherwise the log is from "root" which can be confusing

![image](https://github.com/microsoft/onnxscript/assets/11205048/f04e7158-537e-4eae-ac98-50ee2d9603db)



Discussion:

Resolving patch:
diff --git a/onnxscript/values.py b/onnxscript/values.py
--- a/onnxscript/values.py
+++ b/onnxscript/values.py
@@ -46,6 +46,8 @@
 # A special value to indicate that the default value is not specified
 _EmptyDefault = object()
 
+logger = logging.getLogger(__name__)
+
 
 class Opset:
     """Represents an ONNX Opset, which consists of a domain name, a version.
@@ -106,7 +108,6 @@ def __getattr__(self, attr: str):
 
     def add_function_def(self, fun):
         if fun.name in self.function_defs:
-            logger = logging.getLogger("onnxscript")
             logger.warning("%s: Already defined.", fun.name)
         self.function_defs[fun.name] = fun
 
@@ -284,7 +285,7 @@ def __init__(
         self._param_schemas: Optional[tuple[ParamSchema, ...]] = None
 
         if self._op_schema is None:
-            logging.debug(
+            logger.debug(
                 "An OpSchema was not provided for Op '%s' and "
                 "there is not one found in opset '%s'.",
                 opname,


Test patch:
Repository: microsoft/onnxscript

Issue:
[torchlib] Implement sinc function
A beautiful function useful for signal processing: https://www.sciencedirect.com/topics/engineering/sinc-function#:~:text=The%20Sinc%20Function%20and%20Integration,Hopital's%20rule%20to%20be%20unity.

![image](https://github.com/microsoft/onnxscript/assets/11205048/0e1428fd-a988-42a9-abf6-c0ce1166e3bc)



Discussion:

Resolving patch:
diff --git a/onnxscript/function_libs/torch_lib/ops/special.py b/onnxscript/function_libs/torch_lib/ops/special.py
--- a/onnxscript/function_libs/torch_lib/ops/special.py
+++ b/onnxscript/function_libs/torch_lib/ops/special.py
@@ -11,14 +11,16 @@
 """
 from __future__ import annotations
 
+import math
 from typing import Optional, Sequence
 
 from onnxscript.function_libs.torch_lib.ops import common as common_ops
 from onnxscript.function_libs.torch_lib.registration import torch_op
-from onnxscript.function_libs.torch_lib.tensor_typing import TFloatOrBFloat16
+from onnxscript.function_libs.torch_lib.tensor_typing import TFloat, TFloatOrBFloat16
 from onnxscript.onnx_opset import opset18 as op
 from onnxscript.onnx_types import TensorType
 
+_MATH_PI = math.pi
 IsScalar = common_ops.IsScalar
 
 
@@ -338,10 +340,15 @@ def aten_special_shifted_chebyshev_polynomial_w(x: TensorType, n: TensorType) ->
     raise NotImplementedError()
 
 
-def aten_special_sinc(self: TensorType) -> TensorType:
+@torch_op(("aten::special_sinc", "aten::sinc"))
+def aten_special_sinc(self: TFloat) -> TFloat:
     """special_sinc(Tensor self) -> Tensor"""
 
-    raise NotImplementedError()
+    # This computes the normalized sinc, where the input is multiplied by pi.
+    # https://pytorch.org/docs/stable/special.html#torch.special.sinc
+    pi_self = self * _MATH_PI
+
+    return op.Where(self == 0.0, op.CastLike(1, self), op.Sin(pi_self) / pi_self)
 
 
 def aten_special_spherical_bessel_j0(x: TensorType) -> TensorType:


Test patch:
diff --git a/onnxscript/tests/function_libs/torch_lib/ops_test_data.py b/onnxscript/tests/function_libs/torch_lib/ops_test_data.py
--- a/onnxscript/tests/function_libs/torch_lib/ops_test_data.py
+++ b/onnxscript/tests/function_libs/torch_lib/ops_test_data.py
@@ -1428,6 +1428,9 @@ def _where_input_wrangler(
     TorchLibOpInfo("sigmoid", core_ops.aten_sigmoid),
     TorchLibOpInfo("sign", core_ops.aten_sign),
     TorchLibOpInfo("sin", core_ops.aten_sin),
+    TorchLibOpInfo(
+        "sinc", special_ops.aten_special_sinc, tolerance={torch.float16: (1e-2, 6e-4)}
+    ),
     TorchLibOpInfo("sinh", core_ops.aten_sinh),
     TorchLibOpInfo(
         "softmax",
Repository: deepchecks/deepchecks

Issue:
The mean value is not shown in the regression systematic error plot
I would expect that near the plot (or when I hover over the mean line in the plot), I would see the mean error value.

![image](https://user-images.githubusercontent.com/20860465/147750874-cc7cbc36-bd0d-454a-817f-236f519abfa2.png)

To reproduce:
https://www.kaggle.com/itay94/notebookf8c78e84d7


Discussion:

Resolving patch:
diff --git a/deepchecks/checks/performance/regression_systematic_error.py b/deepchecks/checks/performance/regression_systematic_error.py
--- a/deepchecks/checks/performance/regression_systematic_error.py
+++ b/deepchecks/checks/performance/regression_systematic_error.py
@@ -51,24 +51,19 @@ def _regression_error_distribution(self, dataset: Dataset, model: BaseEstimator)
         diff = y_test - y_pred
         diff_mean = diff.mean()
 
+        fig = go.Figure()
+        fig.add_trace(go.Box(
+            x=diff,
+            name='Model prediction error',
+            boxmean=True  # represent mean
+        )).update_layout(
+                title_text='Box plot of the model prediction error',
+                width=800, height=500)
+
         display = [
             'Non-zero mean of the error distribution indicated the presents '
             'of systematic error in model predictions',
-            go.Figure()
-            .add_trace(go.Box(
-                x=diff.values,
-                name='Model Prediction Error',
-                boxpoints='suspectedoutliers',
-                marker=dict(outliercolor='red'),
-                hoverinfo='x',
-                hoveron='points'))
-            .update_layout(
-                title_text='Box plot of the model prediction error',
-                width=800, height=500)
-            .add_vline(
-                x=diff_mean + 0.01,
-                line_dash='dash',
-                annotation_text='Mean error')
+            fig
         ]
 
         return CheckResult(value={'rmse': rmse, 'mean_error': diff_mean}, display=display)


Test patch:
Repository: deepchecks/deepchecks

Issue:
BUG: Simple model comparison shows irrelevant labels
To reproduce:
https://www.kaggle.com/itay94/notebook05f499eb19

![image](https://user-images.githubusercontent.com/20860465/147876157-58631e60-5110-4f50-b52b-1d4a7432ecc5.png)

![image](https://user-images.githubusercontent.com/20860465/147876162-b6e747b3-5b2f-446f-9c5d-c1fcbd1dffbe.png)



Discussion:

Resolving patch:
diff --git a/deepchecks/checks/performance/simple_model_comparison.py b/deepchecks/checks/performance/simple_model_comparison.py
--- a/deepchecks/checks/performance/simple_model_comparison.py
+++ b/deepchecks/checks/performance/simple_model_comparison.py
@@ -101,10 +101,8 @@ def run(self, train_dataset, test_dataset, model) -> CheckResult:
             for model_name, model_type, model_instance in models:
                 for scorer in scorers:
                     score_result: np.ndarray = scorer(model_instance, test_dataset)
-                    # Multiclass scorers return numpy array of result per class
-                    for class_i, class_score in enumerate(score_result):
-                        # The proba returns in order of the sorted classes.
-                        class_value = train_dataset.classes[class_i]
+                    # Multiclass scorers return numpy array of result per class in order of sorted classes
+                    for class_score, class_value in zip(score_result, test_dataset.classes):
                         results.append([model_name, model_type, class_score, scorer.name, class_value])
 
             results_df = pd.DataFrame(results, columns=['Model', 'Type', 'Value', 'Metric', 'Class'])
@@ -112,7 +110,7 @@ def run(self, train_dataset, test_dataset, model) -> CheckResult:
             # Plot the metrics in a graph, grouping by the model and class
             fig = px.bar(results_df, x=['Class', 'Model'], y='Value', color='Model', barmode='group',
                          facet_col='Metric', facet_col_spacing=0.05)
-            fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60)
+            fig.update_xaxes(title=None, tickprefix='Class ', tickangle=60, type='category')
             fig.update_yaxes(title=None, matches=None)
             fig.for_each_annotation(lambda a: a.update(text=a.text.split('=')[-1]))
             fig.for_each_yaxis(lambda yaxis: yaxis.update(showticklabels=True))


Test patch:
Repository: deepchecks/deepchecks

Issue:
BUG: confusion matrix chart displays wrong output
To reproduce:  https://www.kaggle.com/itay94/notebook05f499eb19

![image](https://user-images.githubusercontent.com/20860465/147875903-9701bdc8-a094-4fb5-807b-d99e66befa37.png)



Discussion:

Resolving patch:
diff --git a/deepchecks/checks/performance/confusion_matrix_report.py b/deepchecks/checks/performance/confusion_matrix_report.py
--- a/deepchecks/checks/performance/confusion_matrix_report.py
+++ b/deepchecks/checks/performance/confusion_matrix_report.py
@@ -9,11 +9,10 @@
 # ----------------------------------------------------------------------------
 #
 """The confusion_matrix_report check module."""
-import numpy as np
 import sklearn
 from sklearn.base import BaseEstimator
 
-import plotly.figure_factory as ff
+import plotly.express as px
 from deepchecks import CheckResult, Dataset
 from deepchecks.base.check import SingleDatasetBaseCheck
 from deepchecks.utils.metrics import ModelType, task_type_validation
@@ -49,15 +48,12 @@ def _confusion_matrix_report(self, dataset: Dataset, model):
         ds_x = dataset.data[dataset.features]
         ds_y = dataset.data[label]
         y_pred = model.predict(ds_x)
-
         confusion_matrix = sklearn.metrics.confusion_matrix(ds_y, y_pred)
 
-        labels = [str(val) for val in np.unique(ds_y)]
-        fig = ff.create_annotated_heatmap(confusion_matrix, x=labels, y=labels, colorscale='Viridis')
+        # Figure
+        fig = px.imshow(confusion_matrix, x=dataset.classes, y=dataset.classes, text_auto=True)
         fig.update_layout(width=600, height=600)
-        fig.update_xaxes(title='Predicted Value')
-        fig.update_yaxes(title='True value', autorange='reversed')
-        fig['data'][0]['showscale'] = True
-        fig['layout']['xaxis']['side'] = 'bottom'
+        fig.update_xaxes(title='Predicted Value', type='category')
+        fig.update_yaxes(title='True value', type='category')
 
         return CheckResult(confusion_matrix, display=fig)


Test patch:
Repository: deepchecks/deepchecks

Issue:
BUG: model info check fails when model is sklearn pipeline
to reproduce:
https://www.kaggle.com/itay94/notebook6f16624759

![image](https://user-images.githubusercontent.com/20860465/147880701-f8edc031-bc19-4a1b-a30b-6f9f8b89b782.png)



Discussion:

Resolving patch:
diff --git a/deepchecks/checks/overview/model_info.py b/deepchecks/checks/overview/model_info.py
--- a/deepchecks/checks/overview/model_info.py
+++ b/deepchecks/checks/overview/model_info.py
@@ -14,6 +14,7 @@
 
 from deepchecks import ModelOnlyBaseCheck, CheckResult
 from deepchecks.utils.validation import model_type_validation
+from deepchecks.utils.model import get_model_of_pipeline
 
 
 __all__ = ['ModelInfo']
@@ -31,13 +32,11 @@ def run(self, model: BaseEstimator) -> CheckResult:
         Returns:
             CheckResult: value is dictionary in format {type: <model_type>, params: <model_params_dict>}
         """
-        return self._model_info(model)
-
-    def _model_info(self, model: BaseEstimator):
         model_type_validation(model)
-        model_type = type(model).__name__
-        model_params = model.get_params()
-        default_params = type(model)().get_params()
+        estimator = get_model_of_pipeline(model)
+        model_type = type(estimator).__name__
+        model_params = estimator.get_params()
+        default_params = type(estimator)().get_params()
 
         # Create dataframe to show
         model_param_df = pd.DataFrame(model_params.items(), columns=['Parameter', 'Value'])


Test patch:
diff --git a/tests/checks/overview/model_info_test.py b/tests/checks/overview/model_info_test.py
--- a/tests/checks/overview/model_info_test.py
+++ b/tests/checks/overview/model_info_test.py
@@ -10,6 +10,9 @@
 #
 """Tests for Model Info."""
 from hamcrest import assert_that, has_entries, calling, raises
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import Pipeline
+
 from deepchecks.checks.overview.model_info import ModelInfo
 from deepchecks.errors import DeepchecksValueError
 
@@ -38,6 +41,16 @@ def test_model_info_object(iris_adaboost):
     assert_model_result(result)
 
 
+def test_model_info_pipeline(iris_adaboost):
+    # Arrange
+    simple_pipeline = Pipeline([('nan_handling', SimpleImputer(strategy='most_frequent')),
+                                ('adaboost', iris_adaboost)])
+    # Act
+    result = ModelInfo().run(simple_pipeline)
+    # Assert
+    assert_model_result(result)
+
+
 def test_model_info_wrong_input():
     # Act
     assert_that(calling(ModelInfo().run).with_args('some string'),
Repository: deepchecks/deepchecks

Issue:
DOCS: broken code blocks in SimpleModelComparison
![image](https://user-images.githubusercontent.com/20860465/147879947-f3a157b1-d464-4672-9774-19203833c7bc.png)



Discussion:

Resolving patch:
diff --git a/deepchecks/checks/performance/model_error_analysis.py b/deepchecks/checks/performance/model_error_analysis.py
--- a/deepchecks/checks/performance/model_error_analysis.py
+++ b/deepchecks/checks/performance/model_error_analysis.py
@@ -64,12 +64,16 @@ class ModelErrorAnalysis(TrainTestBaseCheck):
     For every scorer higher scores are better than lower scores.
 
     You can create a scorer out of existing sklearn metrics:
+
     .. code-block:: python
+
         from sklearn.metrics import roc_auc_score, make_scorer
         auc_scorer = make_scorer(roc_auc_score)
 
     Or you can implement your own:
+
     .. code-block:: python
+
         from sklearn.metrics import make_scorer
 
 
diff --git a/deepchecks/checks/performance/performance_report.py b/deepchecks/checks/performance/performance_report.py
--- a/deepchecks/checks/performance/performance_report.py
+++ b/deepchecks/checks/performance/performance_report.py
@@ -44,12 +44,16 @@ class PerformanceReport(TrainTestBaseCheck):
     For every scorer higher scores are better than lower scores.
 
     You can create a scorer out of existing sklearn metrics:
+
     .. code-block:: python
+
         from sklearn.metrics import roc_auc_score, make_scorer
         auc_scorer = make_scorer(roc_auc_score)
 
     Or you can implement your own:
+
     .. code-block:: python
+
         from sklearn.metrics import make_scorer
 
 
diff --git a/deepchecks/checks/performance/simple_model_comparison.py b/deepchecks/checks/performance/simple_model_comparison.py
--- a/deepchecks/checks/performance/simple_model_comparison.py
+++ b/deepchecks/checks/performance/simple_model_comparison.py
@@ -58,12 +58,16 @@ class SimpleModelComparison(TrainTestBaseCheck):
     For every scorer higher scores are better than lower scores.
 
     You can create a scorer out of existing sklearn metrics:
+
     .. code-block:: python
+
         from sklearn.metrics import roc_auc_score, make_scorer
         auc_scorer = make_scorer(roc_auc_score)
 
     Or you can implement your own:
+
     .. code-block:: python
+
         from sklearn.metrics import make_scorer
 
 


Test patch:
Repository: deepchecks/deepchecks

Issue:
[BUG] unnecessary warnings in integrity suite
Scenario:
When I simply run the example from the readme.

![image](https://user-images.githubusercontent.com/20860465/147885476-3751f540-a0c8-4216-9c68-7cf0f0861fba.png)



Discussion:

Resolving patch:
diff --git a/deepchecks/utils/features.py b/deepchecks/utils/features.py
--- a/deepchecks/utils/features.py
+++ b/deepchecks/utils/features.py
@@ -79,6 +79,8 @@ def calculate_feature_importance_or_none(
             or None if the input is incorrect
     """
     try:
+        if model is None:
+            return None
         # calculate feature importance if dataset has label and the model is fitted on it
         return calculate_feature_importance(
             model=model,


Test patch:
diff --git a/tests/base/feature_importance_utils_test.py b/tests/base/feature_importance_utils_test.py
--- a/tests/base/feature_importance_utils_test.py
+++ b/tests/base/feature_importance_utils_test.py
@@ -10,7 +10,8 @@
 #
 """Test feature importance utils"""
 import pandas as pd
-from hamcrest import equal_to, assert_that, calling, raises, close_to, not_none, none
+import pytest
+from hamcrest import equal_to, assert_that, calling, raises, close_to, not_none, none, has_length
 from sklearn.ensemble import AdaBoostClassifier
 from sklearn.linear_model import LinearRegression, LogisticRegression
 from sklearn.neural_network import MLPClassifier
@@ -110,3 +111,12 @@ def test_fi_n_top(diabetes_split_dataset_and_model):
     sorted_df = column_importance_sorter_df(columns_info_df, train, feature_importances, num_values)
 
     assert_that(list(sorted_df.index), equal_to(feature_importances_sorted))
+
+
+def test_no_warning_on_none_model(iris_dataset):
+    # Act
+    with pytest.warns(None) as warn_record:
+        fi = calculate_feature_importance_or_none(None, iris_dataset)
+    # Assert
+    assert_that(fi, none())
+    assert_that(warn_record, has_length(0))