Repository: NeuroTechX/moabb

Issue:
Results not saved due to issues with get_digest
The `get_digest` function yields duplicate hash values even if two classifier objects have different internal state via different lambda functions. https://github.com/NeuroTechX/moabb/blob/fd0ffe66572a273a6a2d19174a21cd9c4e4e94c9/moabb/analysis/results.py#L111
Short example:

```python
pipeline_a = FancyClassifier(kernel=lambda d: 1/d**2)
pipeline_b = FancyClassifier(kernel=lambda d: 1/d**4)
```

As of now, after running moabb the results file / dataframe would contain 2 scores for `pipeline_b` as the `get_digest` returns the equivalent hash for both pipelines. (The memory addresses identifying the different lambda functions are wiped, but I do not really know why)

Is there a reason moabb does not simply use the pipeline description string? Or just the hash thereof? I.e.:

```python
digest = get_digest(name) 
```

Discussion:
sylvchev: As this seems not possible to identify the error on windows, I think you could modify the unit test to skip the test if it is on Windows. This is a bad practice, but in this specific case, the error came from python memory representation, thus we can indulge this skip.
sylvchev: Let's merge this!

Patch:
diff --git a/moabb/analysis/results.py b/moabb/analysis/results.py
index 03f378dc..eb117865 100644
--- a/moabb/analysis/results.py
+++ b/moabb/analysis/results.py
@@ -2,6 +2,7 @@ import hashlib
 import os
 import os.path as osp
 import re
+import warnings
 from datetime import datetime
 
 import h5py
@@ -17,6 +18,15 @@ def get_string_rep(obj):
         str_repr = repr(obj.get_params())
     else:
         str_repr = repr(obj)
+    if "<lambda> at " in str_repr:
+        warnings.warn(
+            "You are probably using a classifier with a lambda function"
+            " as an attribute. Lambda functions can only be identified"
+            " by memory address which MOABB does not consider. To avoid"
+            " issues you can use named functions defined using the def"
+            " keyword instead.",
+            RuntimeWarning,
+        )
     str_no_addresses = re.sub("0x[a-z0-9]*", "0x__", str_repr)
     return str_no_addresses.replace("\n", "").encode("utf8")
 


Test patch:
diff --git a/moabb/tests/evaluations.py b/moabb/tests/evaluations.py
index d42a48a1..e4bc6648 100644
--- a/moabb/tests/evaluations.py
+++ b/moabb/tests/evaluations.py
@@ -1,14 +1,18 @@
 import os
 import os.path as osp
+import platform
 import unittest
+import warnings
 from collections import OrderedDict
 
 import numpy as np
+import sklearn.base
 from pyriemann.estimation import Covariances
 from pyriemann.spatialfilters import CSP
 from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
 from sklearn.pipeline import make_pipeline
 
+from moabb.analysis.results import get_string_rep
 from moabb.datasets.fake import FakeDataset
 from moabb.evaluations import evaluations as ev
 from moabb.paradigms.motor_imagery import FakeImageryParadigm
@@ -21,6 +25,13 @@ if not osp.isdir(osp.join(osp.expanduser("~"), "mne_data")):
     os.makedirs(osp.join(osp.expanduser("~"), "mne_data"))
 
 
+class DummyClassifier(sklearn.base.BaseEstimator):
+    __slots__ = "kernel"
+
+    def __init__(self, kernel):
+        self.kernel = kernel
+
+
 class Test_WithinSess(unittest.TestCase):
     """This is actually integration testing but I don't know how to do this
     better. A paradigm implements pre-processing so it needs files to run MNE
@@ -177,5 +188,31 @@ class Test_CrossSess(Test_WithinSess):
         self.assertTrue(self.eval.is_valid(dataset=ds))
 
 
+class Test_LambdaWarning(Test_WithinSess):
+    def setUp(self):
+        self.eval = ev.WithinSessionEvaluation(
+            paradigm=FakeImageryParadigm(), datasets=[dataset]
+        )
+
+    def test_lambda_warning(self):
+        def explicit_kernel(x):
+            return x ** 3
+
+        c1 = DummyClassifier(kernel=lambda x: x ** 2)
+        c2 = DummyClassifier(kernel=lambda x: 5 * x)
+
+        c3 = DummyClassifier(kernel=explicit_kernel)
+
+        self.assertFalse(repr(c1) == repr(c2))
+        if platform.system() != "Windows":
+            with self.assertWarns(RuntimeWarning):
+                self.assertTrue(get_string_rep(c1) == get_string_rep(c2))
+
+        # I do not know an elegant way to check for no warnings
+        with warnings.catch_warnings(record=True) as w:
+            get_string_rep(c3)
+            self.assertTrue(len(w) == 0)
+
+
 if __name__ == "__main__":
     unittest.main()