Repository: microsoft/pybryt

Issue:
Add a public API for checking values against Value annotations
As titled. For example, to check whether a value annotation is satisfied by some object:

```python
v = pybryt.Value(obj)
v.check_against(other_obj)
```

Allows instructors to test the robustness of their annotations and engage in unit testing assignments. Also useful for demonstration purposes.

Any method implementing this feature should just wrap the object in the tuple abstraction for a memory footprint observed value and use existing methods to validate the pseudo-footprint.

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 72bb725..7df7195 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,12 @@
 All notable changes to this project will be documented in this file, and this project adheres to 
 [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+* Fixed bug in `Value`s tracking `set`s of numbers
+* Fixed bug in timestamp comparison in `BeforeAnnotation`
+* Added `Value.check_against` and `Attribute.check_agains` per [#111](https://github.com/microsoft/pybryt/issues/111)
+
 ## 0.2.0 2021-09-01
 
 * Fixed - 'Empty iterable comparison' per [#109](https://github.com/microsoft/pybryt/pull/109)
diff --git a/pybryt/annotations/relation.py b/pybryt/annotations/relation.py
index 5870fd9..676099e 100644
--- a/pybryt/annotations/relation.py
+++ b/pybryt/annotations/relation.py
@@ -103,7 +103,7 @@ class BeforeAnnotation(RelationalAnnotation):
         if all(res.satisfied for res in results):
             before = []
             for i in range(len(self._annotations) - 1):
-                before.append(results[i].satisfied_at < results[i + 1].satisfied_at)
+                before.append(results[i].satisfied_at <= results[i + 1].satisfied_at)
             
             return AnnotationResult(all(before), self, children = results)
         
diff --git a/pybryt/annotations/value.py b/pybryt/annotations/value.py
index 62d6bb9..9957977 100644
--- a/pybryt/annotations/value.py
+++ b/pybryt/annotations/value.py
@@ -7,7 +7,7 @@ import numbers
 import pandas as pd
 import numpy as np
 
-from collections.abc import Iterable
+from collections.abc import Iterable, Sized
 from copy import copy
 from typing import Any, Dict, List, Optional, Tuple, Union
 
@@ -139,6 +139,18 @@ class Value(Annotation):
             self.check_values_equal(self.initial_value, other.initial_value) and \
             self.atol == other.atol and self.rtol == other.rtol
 
+    def check_against(self, other_value: Any) -> bool:
+        """
+        Check whether an object satisfies this annotation.
+
+        Args:
+            other_value (``object``): the value to check against
+
+        Returns:
+            ``bool``: whether this annotation is satisfied by the provided value
+        """
+        return self.check([(other_value, 0)]).satisfied
+
     def _check_observed_value(self, observed_value: Tuple[Any, int]) -> bool:
         """
         Checks whether a single observed value tuple satisfies this value.
@@ -212,14 +224,25 @@ class Value(Annotation):
                 return False
         else:
             try:
-                if (hasattr(value, "shape") and hasattr(other_value, "shape") and value.shape != other_value.shape) \
+                if (hasattr(value, "shape") and hasattr(other_value, "shape") \
+                        and value.shape != other_value.shape) \
                         or (hasattr(value, "shape") ^ hasattr(other_value, "shape")):
                     return False
-                if len(value) > 0 and isinstance(value, Iterable) and all(isinstance(i, numbers.Real) for i in value):
-                    # Tolerances make sense only for iterables with numerical data.
-                    res = np.allclose(value, other_value, atol=atol, rtol=rtol)
+
+                # tolerances make sense only for iterables with numerical data
+                if isinstance(value, Sized) and len(value) > 0 and isinstance(value, Iterable) \
+                        and all(isinstance(i, numbers.Real) for i in value):
+                    # np.allclose doesn't work with sets
+                    if isinstance(value, set):
+                        if len(value) != len(other_value):
+                            return False
+                        res = np.array(np.isclose(v, o, atol=atol, rtol=rtol) for v, o in zip(sorted(value), sorted(other_value))).all()
+                    else:
+                        res = np.allclose(value, other_value, atol=atol, rtol=rtol)
+
                 else:
                     res = value == other_value
+
             except (ValueError, TypeError) as e:
                 return False
 
@@ -414,3 +437,15 @@ class Attribute(Annotation):
         """
         results = [v.check(observed_values) for v in self._annotations]        
         return AnnotationResult(None, self, children=results)
+
+    def check_against(self, other_value: Any) -> bool:
+        """
+        Check whether an object satisfies this annotation.
+
+        Args:
+            other_value (``object``): the value to check against
+
+        Returns:
+            ``bool``: whether this annotation is satisfied by the provided value
+        """
+        return self.check([(other_value, 0)]).satisfied


Test patch:
diff --git a/tests/annotations/test_value.py b/tests/annotations/test_value.py
index e6f77a2..62ee914 100644
--- a/tests/annotations/test_value.py
+++ b/tests/annotations/test_value.py
@@ -83,6 +83,13 @@ def test_value_annotation():
     res = v.check(mfp)
     assert res.satisfied
 
+    # test that check_against correctly calls check
+    with mock.patch.object(v, "check") as mocked_check:
+        mocked_check.return_value = mock.MagicMock()
+        mocked_check.return_value.satisfied = True
+        assert v.check_against(s.lower())
+        mocked_check.assert_called_with([(s.lower(), 0)])
+
 
 def test_attribute_annotation():
     """
@@ -135,6 +142,13 @@ def test_attribute_annotation():
         "enforce_type": False,
     }
 
+    # test that check_against correctly calls check
+    with mock.patch.object(v, "check") as mocked_check:
+        mocked_check.return_value = mock.MagicMock()
+        mocked_check.return_value.satisfied = False
+        assert not v.check_against(val)
+        mocked_check.assert_called_with([(val, 0)])
+
     # check enforce type
     class Foo:
         T = val.T