Repository: biolab/orange3

Issue:
Random: Cannot copy model with Impute preprocessor
<!-- 
Thanks for taking the time to report a bug!
If you're raising an issue about an add-on (i.e., installed via Options > Add-ons), raise an issue in the relevant add-on's issue tracker instead. See: https://github.com/biolab?q=orange3
To fix the bug, we need to be able to reproduce it. Please answer the following questions to the best of your ability.
-->


- [ ] **What's wrong?**
<!-- Be specific, clear, and concise. Include screenshots if relevant. -->
<!-- If you're getting an error message, copy it, and enclose it with three backticks (```). -->
Model with `Impute(method=Random())` preprocessor can not be deepcopied.



- [ ] **How can we reproduce the problem?** 
<!-- Upload a zip with the .ows file and data. -->
<!-- Describe the steps (open this widget, click there, then add this...) -->
Run the following snippet.
```
import copy
from Orange.data import Table
from Orange.preprocess import Impute
from Orange.preprocess.impute import Random
from Orange.regression import LinearRegressionLearner

learner = LinearRegressionLearner(
    preprocessors=Impute(method=Random())
)
copy.deepcopy(learner(Table("housing")))
```



- [ ] **What's your environment?**
<!-- To find your Orange version, see "Help → About → Version" or `Orange.version.full_version` in code -->
- Operating system: MacOS
- Orange version: master
- How you installed Orange: source



Discussion:
`Random` contains `Continuous`, which is derived from `Distribution` and, in turn, from `np.ndarray`. The problem is that

```python
import numpy as np
import pickle

class A(np.ndarray):
    pass

a = A([1, 2, 3])
a.x = 12

b = pickle.loads(pickle.dumps(a))
b.x
```

fails with 

```
AttributeError: 'A' object has no attribute 'x'
```

because `ndarray` implements `__reduce__` and does not copy attributes added in derived classes.

We will need to implement `__reduce__` for all classes derived from `ndarray`. Deriving from `ndarray` is nothing but trouble, and we should have avoided it before it was too late.

Alternatively, we should check whether `Distribution` and `Contingency` could be refactored without breaking too much code. These classes are about as useless and thus as unused as `Instance` and `RowInstance`.
> We will need to implement `__reduce__` for all classes derived from `ndarray`.

... as well as `__deepcopy__`.

> Deriving from ndarray is nothing but trouble, and we should have avoided it before it was too late.

When I die and go to purgatory, I won't do anything special; I'll atone for my sins by continuing to maintain them.

Resolving patch:
diff --git a/Orange/statistics/contingency.py b/Orange/statistics/contingency.py
--- a/Orange/statistics/contingency.py
+++ b/Orange/statistics/contingency.py
@@ -179,9 +179,20 @@ def __reduce__(self):
         return (
             _create_discrete,
             (Discrete, np.copy(self), self.col_variable, self.row_variable,
-             self.col_unknowns, self.row_unknowns)
+             self.col_unknowns, self.row_unknowns, self.unknowns)
         )
 
+    def __array_finalize__(self, obj):
+        # defined in __new__, pylint: disable=attribute-defined-outside-init
+        """See http://docs.scipy.org/doc/numpy/user/basics.subclassing.html"""
+        if obj is None:
+            return
+        self.col_variable = getattr(obj, 'col_variable', None)
+        self.row_variable = getattr(obj, 'row_variable', None)
+        self.col_unknowns = getattr(obj, 'col_unknowns', None)
+        self.row_unknowns = getattr(obj, 'row_unknowns', None)
+        self.unknowns = getattr(obj, 'unknowns', None)
+
 
 class Continuous:
     def __init__(self, dat, col_variable=None, row_variable=None,
diff --git a/Orange/statistics/distribution.py b/Orange/statistics/distribution.py
--- a/Orange/statistics/distribution.py
+++ b/Orange/statistics/distribution.py
@@ -30,6 +30,26 @@ def _get_variable(dat, variable, expected_type=None, expected_name=""):
 
 
 class Distribution(np.ndarray):
+    def __array_finalize__(self, obj):
+        # defined in derived classes,
+        # pylint: disable=attribute-defined-outside-init
+        """See http://docs.scipy.org/doc/numpy/user/basics.subclassing.html"""
+        if obj is None:
+            return
+        self.variable = getattr(obj, 'variable', None)
+        self.unknowns = getattr(obj, 'unknowns', 0)
+
+    def __reduce__(self):
+        state = super().__reduce__()
+        newstate = state[2] + (self.variable, self.unknowns)
+        return state[0], state[1], newstate
+
+    def __setstate__(self, state):
+        # defined in derived classes,
+        # pylint: disable=attribute-defined-outside-init
+        super().__setstate__(state[:-2])
+        self.variable, self.unknowns = state[-2:]
+
     def __eq__(self, other):
         return (
             np.array_equal(self, other) and


Test patch:
diff --git a/Orange/tests/test_contingency.py b/Orange/tests/test_contingency.py
--- a/Orange/tests/test_contingency.py
+++ b/Orange/tests/test_contingency.py
@@ -1,6 +1,6 @@
 # Test methods with long descriptive names can omit docstrings
 # pylint: disable=missing-docstring
-
+import copy
 import unittest
 from unittest.mock import Mock
 
@@ -71,6 +71,13 @@ def test_discrete_missing(self):
         np.testing.assert_almost_equal(cont.row_unknowns, [0, 0])
         self.assertEqual(1, cont.unknowns)
 
+    def test_deepcopy(self):
+        cont = contingency.Discrete(self.zoo, 0)
+        dc = copy.deepcopy(cont)
+        self.assertEqual(dc, cont)
+        self.assertEqual(dc.col_variable, cont.col_variable)
+        self.assertEqual(dc.row_variable, cont.row_variable)
+
     def test_array_with_unknowns(self):
         d = data.Table("zoo")
         d.Y[2] = float("nan")
diff --git a/Orange/tests/test_distribution.py b/Orange/tests/test_distribution.py
--- a/Orange/tests/test_distribution.py
+++ b/Orange/tests/test_distribution.py
@@ -1,7 +1,8 @@
 # Test methods with long descriptive names can omit docstrings
 # Test internal methods
 # pylint: disable=missing-docstring, protected-access
-
+import copy
+import pickle
 import unittest
 from unittest.mock import Mock
 import warnings
@@ -110,6 +111,32 @@ def test_fallback_with_weights_and_nan(self):
             np.asarray(fallback), np.asarray(default))
         np.testing.assert_almost_equal(fallback.unknowns, default.unknowns)
 
+    def test_pickle(self):
+        d = data.Table("zoo")
+        d1 = distribution.Discrete(d, 0)
+        dc = pickle.loads(pickle.dumps(d1))
+        # This always worked because `other` wasn't required to have `unknowns`
+        self.assertEqual(d1, dc)
+        # This failed before implementing `__reduce__`
+        self.assertEqual(dc, d1)
+        self.assertEqual(hash(d1), hash(dc))
+        # Test that `dc` has the required attributes
+        self.assertEqual(dc.variable, d1.variable)
+        self.assertEqual(dc.unknowns, d1.unknowns)
+
+    def test_deepcopy(self):
+        d = data.Table("zoo")
+        d1 = distribution.Discrete(d, 0)
+        dc = copy.deepcopy(d1)
+        # This always worked because `other` wasn't required to have `unknowns`
+        self.assertEqual(d1, dc)
+        # This failed before implementing `__deepcopy__`
+        self.assertEqual(dc, d1)
+        self.assertEqual(hash(d1), hash(dc))
+        # Test that `dc` has the required attributes
+        self.assertEqual(dc.variable, d1.variable)
+        self.assertEqual(dc.unknowns, d1.unknowns)
+
     def test_equality(self):
         d = data.Table("zoo")
         d1 = distribution.Discrete(d, 0)
@@ -285,6 +312,30 @@ def test_construction(self):
         self.assertEqual(disc2.unknowns, 0)
         assert_dist_equal(disc2, dd)
 
+    def test_pickle(self):
+        d1 = distribution.Continuous(self.iris, 0)
+        dc = pickle.loads(pickle.dumps(d1))
+        # This always worked because `other` wasn't required to have `unknowns`
+        self.assertEqual(d1, dc)
+        # This failed before implementing `__reduce__`
+        self.assertEqual(dc, d1)
+        self.assertEqual(hash(d1), hash(dc))
+        # Test that `dc` has the required attributes
+        self.assertEqual(dc.variable, d1.variable)
+        self.assertEqual(dc.unknowns, d1.unknowns)
+
+    def test_deepcopy(self):
+        d1 = distribution.Continuous(self.iris, 0)
+        dc = copy.deepcopy(d1)
+        # This always worked because `other` wasn't required to have `unknowns`
+        self.assertEqual(d1, dc)
+        # This failed before implementing `__deepcopy__`
+        self.assertEqual(dc, d1)
+        self.assertEqual(hash(d1), hash(dc))
+        # Test that `dc` has the required attributes
+        self.assertEqual(dc.variable, d1.variable)
+        self.assertEqual(dc.unknowns, d1.unknowns)
+
     def test_hash(self):
         d = self.iris
         petal_length = d.columns.petal_length