Repository: Apkawa/pytest-image-diff (language: python)

Issue:
Check if images are different
* pytest-image-diff version: 0.0.8
* Python version: 3.1
* Operating System: macOS Monterey

### Description
Executed
```image_diff(path_to_image_before, path_to_image_after)```
if images are equal error is not thrown, but if the threshold is reached and images are not equal assertation error is thrown "AssertionError: Image not equals!"
Is there a way to check if images are not equal or threshold is _bigger than x_?

I guess I would expect to do something like
```assert image_diff(path_to_image_before, path_to_image_after)```
or just prefix with ```not``` to get the opposite
```assert not image_diff(path_to_image_before, path_to_image_after)```

Thank you in advance! 



Discussion / hints:
Hello
Okay, I also think throwing an exception in a function is a bad idea.
I think implement like as

```python
import pytest

from typing import Union
from PIL import Image

@pytest.fixture(scope="session")
def image_diff_throw_exception() -> bool:
    """
    Set default throw exception. By default - True
    """
    return False

def test_compare(image_diff):
    image: Image or str or bytes = Image.new()
    image2: Image or str or bytes = '/path/to/image.jpeg'
    assert image_diff(image, image2)
    assert image_diff(image, image2, threshold=0.5)
    # Also can check threshold in compare, ie
    assert image_diff(image, image2) < 0.5
    # For different checks in one test
    assert image_diff(image, image2, threshold=0.5, suffix="one")
    # Or without fixture image_diff_throw_exception
    assert image_diff(image, image2, threshold=0.5, throw_exception=False)


def test_regression(image_regression):
    image: Union[Image, str, bytes] = Image.new()
    assert image_regression(image, threshold=0.5)
    # Also can check threshold in compare, ie
    assert image_regression(image) < 0.5
    # For different checks in one test
    assert image_regression(image, threshold=0.5, suffix="foo")
    # Or without fixture image_diff_throw_exception
    assert image_regression(image, threshold=0.5, throw_exception=False)
```

ie for backward compatible add fixture for disabled exception in function. 
in future think maybe make deprecated and turn off exception in next releases


Fix patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f2148d2..555cb38 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,20 +3,17 @@ name: ci
 on:
   push:
   pull_request:
-  schedule:
-    # Weekly
-    - cron: "0 0 * * 0"
 
 jobs:
   test:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04]
-        python-version: [ 3.7, 3.8, 3.9, "3.10"]
-        toxenv: [""]
+        os: [ ubuntu-20.04 ]
+        python-version: [ 3.7, 3.8, 3.9, "3.10" ]
+        toxenv: [ "" ]
         experimental: [ false ]
-        chrome: ["stable"]
+        chrome: [ "stable" ]
         include:
           - toxenv: qa
             python-version: 3.7
@@ -36,7 +33,7 @@ jobs:
             experimental: true
             python-version: 3.7
             os: windows-latest
-            chrome: "959905"
+#            chrome: "959905"
 
           - experimental: false
             python-version: "3.9"
diff --git a/README.md b/README.md
index a69fe5f..0ac4afe 100644
--- a/README.md
+++ b/README.md
@@ -40,6 +40,45 @@ def test_regression(image_regression):
     image_regression(image, threshold=0.5)
 ```
 
+Also use with assert
+
+```python
+import pytest
+
+from typing import Union
+from PIL import Image
+
+@pytest.fixture(scope="session")
+def image_diff_throw_exception() -> bool:
+    """
+    Set default throw exception. By default - True
+    """
+    return False
+
+def test_compare(image_diff):
+    image: Image or str or bytes = Image.new()
+    image2: Image or str or bytes = '/path/to/image.jpeg'
+    assert image_diff(image, image2)
+    assert image_diff(image, image2, threshold=0.5)
+    # Also can check threshold in compare, ie
+    assert image_diff(image, image2) < 0.5
+    # For different checks in one test
+    assert image_diff(image, image2, threshold=0.5, suffix="one")
+    # Or without fixture image_diff_throw_exception
+    assert image_diff(image, image2, threshold=0.5, throw_exception=False)
+
+
+def test_regression(image_regression):
+    image: Union[Image, str, bytes] = Image.new()
+    assert image_regression(image, threshold=0.5)
+    # Also can check threshold in compare, ie
+    assert image_regression(image) < 0.5
+    # For different checks in one test
+    assert image_regression(image, threshold=0.5, suffix="foo")
+    # Or without fixture image_diff_throw_exception
+    assert image_regression(image, threshold=0.5, throw_exception=False)
+```
+
 First run creates reference images
 
 ## pytest-splinter
diff --git a/docs/source/code.rst b/docs/source/code.rst
index fe3c117..3ed036a 100644
--- a/docs/source/code.rst
+++ b/docs/source/code.rst
@@ -10,6 +10,8 @@ Config fixtures
 .. autofunction:: pytest_image_diff.plugin.image_diff_dir
 .. autofunction:: pytest_image_diff.plugin.image_diff_reference_dir
 
+.. autofunction:: pytest_image_diff.plugin.image_diff_throw_exception
+
 
 Fixtures
 --------
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 4296a12..4487a55 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -5,8 +5,8 @@ tox
 twine
 
 # Tests
-pytest>=6,<7
-pytest-cov==2.12.1
+pytest
+pytest-cov
 
 # Docs
 
diff --git a/setup.cfg b/setup.cfg
index 0d81f13..18c2590 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -20,7 +20,7 @@ universal = 1
 
 [flake8]
 ignore = D203
-exclude = 
+exclude =
 	.git/,
 	.tox/,
 	docs/,
@@ -29,7 +29,7 @@ exclude =
 max-line-length = 99
 
 [mypy]
-python_version = 3.6
+python_version = 3.7
 ignore_missing_imports = True
 warn_return_any = True
 warn_unused_configs = True
diff --git a/tox.ini b/tox.ini
index 62c67a7..ffc2b73 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,7 @@ deps =
     -r{toxinidir}/requirements-dev.txt
     splinter: -e .[splinter]
     splinter-linux: chromedriver-binary-auto
-    splinter-windows: chromedriver-binary==95.0.4638.69.0
+    splinter-windows: chromedriver-binary
 
 setenv =
     PYTHONPATH = {toxinidir}


Test patch:
diff --git a/pytest_image_diff/__init__.py b/pytest_image_diff/__init__.py
index aadc7a5..a9167b9 100644
--- a/pytest_image_diff/__init__.py
+++ b/pytest_image_diff/__init__.py
@@ -1,3 +1,6 @@
 # -*- coding: utf-8 -*-
+import pytest
+
+pytest.register_assert_rewrite("pytest_image_diff.helpers")
 
 __version__ = "0.0.8"
diff --git a/pytest_image_diff/_types.py b/pytest_image_diff/_types.py
index a5621c5..3bba70e 100644
--- a/pytest_image_diff/_types.py
+++ b/pytest_image_diff/_types.py
@@ -1,8 +1,11 @@
 from pathlib import Path
-from typing import BinaryIO, Union, Tuple, Optional
+from typing import BinaryIO, Union, Tuple, Optional, TYPE_CHECKING
+from typing_extensions import Literal, Protocol
 
 from PIL.Image import Image
-from typing_extensions import Literal, Protocol
+
+if TYPE_CHECKING:
+    from .plugin import DiffCompareResult
 
 PathOrFileType = Union[str, bytes, Path, BinaryIO]
 ImageFileType = Union[Image, PathOrFileType]
@@ -15,7 +18,8 @@ def __call__(
         image: ImageFileType,
         threshold: Optional[float] = None,
         suffix: Optional[str] = None,
-    ) -> bool:
+        throw_exception: Optional[bool] = None,
+    ) -> "DiffCompareResult":
         pass
 
 
@@ -26,7 +30,8 @@ def __call__(
         image_2: ImageFileType,
         threshold: Optional[float] = None,
         suffix: Optional[str] = None,
-    ) -> bool:
+        throw_exception: Optional[bool] = None,
+    ) -> "DiffCompareResult":
         pass
 
 
diff --git a/pytest_image_diff/helpers.py b/pytest_image_diff/helpers.py
index 432fd7c..a27aa5e 100644
--- a/pytest_image_diff/helpers.py
+++ b/pytest_image_diff/helpers.py
@@ -38,9 +38,13 @@ def temp_file(suffix: typing.Optional[str] = None) -> typing.Iterator[pathlib.Pa
         yield temp_image_path
     finally:
         try:
+            tf.close()
             temp_image_path.unlink()
         except FileNotFoundError:
             pass
+        except PermissionError:
+            # Opps, windows issue
+            pass
 
 
 class TestInfo:
@@ -96,3 +100,8 @@ def ensure_dirs(filepath: str) -> None:
         file_dir = os.path.dirname(filepath)
         if not os.path.exists(file_dir):
             os.makedirs(file_dir)
+
+
+# def pytest_assertrepr_compare(config, op, left, right):
+#     print(op, left, right)
+#     pass
diff --git a/pytest_image_diff/plugin.py b/pytest_image_diff/plugin.py
index 6b2d5b3..7e19bee 100644
--- a/pytest_image_diff/plugin.py
+++ b/pytest_image_diff/plugin.py
@@ -1,5 +1,5 @@
 import os
-from typing import Optional, NamedTuple, cast, Callable, Generator
+from typing import Optional, NamedTuple, cast, Callable, Generator, Union
 
 import pytest
 from _pytest.fixtures import FixtureRequest
@@ -36,12 +36,20 @@ def image_diff_threshold() -> float:
     return 0.001
 
 
+@pytest.fixture(scope="session")
+def image_diff_throw_exception() -> bool:
+    """
+    Set default throw exception. By default - True
+    """
+    return True
+
+
 @pytest.fixture(scope="session")
 def image_diff_root(request: FixtureRequest) -> str:
     """
     Root path for storing diff images. By default - `request.config.rootdir`
     """
-    return str(request.config.rootdir)
+    return str(request.config.rootdir)  # type: ignore
 
 
 @pytest.fixture(scope="session")  # pragma: no cover
@@ -69,6 +77,37 @@ class DiffInfo(NamedTuple):
 DiffInfoCallableType = Callable[[ImageFileType, Optional[str]], DiffInfo]
 
 
+class DiffCompareResult:
+    value: float
+    thresh: float
+    diff_path: Optional[str]
+
+    CompareType = Union[int, float, "DiffCompareResult"]
+
+    def __init__(self, value: float, thresh: float, diff_path: Optional[str] = None):
+        self.value = value
+        self.thresh = thresh
+        self.diff_path = diff_path
+
+    def __repr__(self) -> str:
+        return f"diff: {round(self.value, 4)}; thresh: {self.thresh}; diff path: {self.diff_path}"
+
+    def __bool__(self) -> bool:
+        return self.value <= self.thresh
+
+    def __gt__(self, other: CompareType) -> bool:
+        return self.value > other
+
+    def __ge__(self, other: CompareType) -> bool:
+        return self.value >= other
+
+    def __lt__(self, other: CompareType) -> bool:
+        return self.value < other
+
+    def __le__(self, other: CompareType) -> bool:
+        return self.value <= other
+
+
 @pytest.fixture(scope="function")
 def _image_diff_info(
     request: FixtureRequest, image_diff_reference_dir: str, image_diff_dir: str
@@ -108,6 +147,7 @@ def image_regression(
     request: FixtureRequest,
     _image_diff_info: DiffInfoCallableType,
     image_diff_threshold: float,
+    image_diff_throw_exception: bool,
 ) -> Generator[ImageRegressionCallableType, None, None]:
     """
     Check regression image.
@@ -122,9 +162,12 @@ def _factory(
         image: ImageFileType,
         threshold: Optional[float] = None,
         suffix: Optional[str] = None,
-    ) -> bool:
+        throw_exception: Optional[bool] = None,
+    ) -> "DiffCompareResult":
         if threshold is None:
             threshold = image_diff_threshold
+        if throw_exception is None:
+            throw_exception = image_diff_throw_exception
         diff_info = _image_diff_info(image, suffix)
         reference_name = diff_info.reference_name
         diff_name = diff_info.diff_name
@@ -132,7 +175,7 @@ def _factory(
         if not os.path.exists(reference_name):
             ensure_dirs(reference_name)
             image_save(image, reference_name)
-            return True
+            return DiffCompareResult(0, threshold)
 
         def _cleanup() -> None:
             if request.node.rep_setup.passed and request.node.rep_call.failed:
@@ -140,7 +183,11 @@ def _cleanup() -> None:
                 return
             for f in [image_name, diff_name]:
                 if os.path.exists(f):
-                    os.unlink(f)
+                    try:
+                        os.unlink(f)
+                    except PermissionError:
+                        # Opps, have windows issue
+                        pass
 
         request.addfinalizer(_cleanup)
 
@@ -148,9 +195,10 @@ def _cleanup() -> None:
         ensure_dirs(image_name)
         image_save(image, image_name)
         diff_ratio = _diff(reference_name, image_name, diff_name)
-        assert diff_ratio <= threshold, "Image not equals!"  # noqa
-
-        return True
+        cond = DiffCompareResult(diff_ratio, threshold, diff_name)
+        if throw_exception:
+            assert cond, "Image not equals!"  # noqa
+        return cond
 
     yield _factory
 
@@ -160,6 +208,7 @@ def image_diff(
     request: FixtureRequest,
     _image_diff_info: DiffInfoCallableType,
     image_diff_threshold: float,
+    image_diff_throw_exception: bool,
 ) -> Generator[ImageDiffCallableType, None, None]:
     """
     Compare two image
@@ -176,9 +225,12 @@ def _factory(
         image_2: ImageFileType,
         threshold: Optional[float] = None,
         suffix: Optional[str] = None,
-    ) -> bool:
+        throw_exception: Optional[bool] = None,
+    ) -> DiffCompareResult:
         if threshold is None:
             threshold = image_diff_threshold
+        if throw_exception is None:
+            throw_exception = image_diff_throw_exception
 
         _info = _image_diff_info(image, suffix)
         diff_path = cast(str, _info.diff_name)
@@ -201,7 +253,9 @@ def _cleanup() -> None:
             image_save(image, path=image_temp_file)
             image_save(image_2, path=image_2_temp_file)
             diff_ratio = _diff(image_temp_file, image_2_temp_file, diff_path)
-        assert diff_ratio <= threshold, "Image not equals! See %s" % diff_path  # noqa
-        return True
+        cond = DiffCompareResult(diff_ratio, threshold, diff_path)
+        if throw_exception:
+            assert cond, "Image not equals! See %s" % diff_path  # noqa
+        return cond
 
     yield _factory
diff --git a/pytest_image_diff/splinter.py b/pytest_image_diff/splinter.py
index 86678da..dff87fd 100644
--- a/pytest_image_diff/splinter.py
+++ b/pytest_image_diff/splinter.py
@@ -1,11 +1,14 @@
 import os
-from typing import Optional, Generator
+from typing import Optional, Generator, TYPE_CHECKING
 
 import pytest
 from typing_extensions import Protocol
 
 from .helpers import temp_file
 
+if TYPE_CHECKING:
+    from .plugin import DiffCompareResult
+
 try:
     # Check pytest-splinter
     from pytest_splinter.plugin import Browser
@@ -15,6 +18,8 @@
 if Browser:
     from ._types import ImageRegressionCallableType
 
+    __all__ = ["screenshot_regression"]
+
     class ScreenshotRegressionCallableType(Protocol):
         def __call__(
             self,
@@ -22,16 +27,15 @@ def __call__(
             threshold: Optional[float] = None,
             suffix: Optional[str] = None,
             xpath: Optional[str] = "",
-        ) -> bool:
+        ) -> "DiffCompareResult":
             pass
 
-    __all__ = ["screenshot_regression"]
-
     @pytest.fixture(scope="function")
     def screenshot_regression(
         browser: Browser,
         image_regression: ImageRegressionCallableType,
         image_diff_threshold: float,
+        image_diff_throw_exception: bool,
     ) -> Generator[ScreenshotRegressionCallableType, None, None]:
         """
         Check regression browser screenshot
@@ -49,13 +53,17 @@ def _factory(
             threshold: Optional[float] = None,
             suffix: Optional[str] = "",
             xpath: Optional[str] = "",
-        ) -> bool:
+            throw_exception: Optional[bool] = None,
+        ) -> "DiffCompareResult":
             if browser is None:
                 browser = default_browser
 
             if threshold is None:
                 threshold = image_diff_threshold
 
+            if throw_exception is None:
+                throw_exception = image_diff_throw_exception
+
             with temp_file(suffix=".png") as temp_image_path:
                 screenshot_path = os.fspath(temp_image_path)
 
@@ -75,7 +83,9 @@ def _factory(
                 else:
                     browser.driver.save_screenshot(screenshot_path)
 
-                result = image_regression(screenshot_path, threshold, suffix)
+                result = image_regression(
+                    screenshot_path, threshold, suffix, throw_exception=throw_exception
+                )
                 return result
 
         yield _factory
diff --git a/tests/conftest.py b/tests/conftest.py
index db23601..397128c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -9,4 +9,4 @@ def image_diff_root(request: FixtureRequest) -> str:
     """
     Root path for storing diff images. By default - `request.config.rootdir`
     """
-    return os.path.join(request.config.rootdir, "tests")
+    return os.path.join(request.config.rootdir, "tests")  # type: ignore
diff --git a/tests/test_diff.py b/tests/test_diff.py
index 466ce72..2061cf3 100644
--- a/tests/test_diff.py
+++ b/tests/test_diff.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-import os
-import shutil
+import random
+import string
 
 import pytest
 from PIL import Image, ImageDraw
@@ -16,10 +16,9 @@ def make_test_image(text="Hello world", size=(100, 30)) -> Image:
 
 
 def test_initial_diff(image_diff, image_diff_dir):
-    if os.path.exists(image_diff_dir):
-        shutil.rmtree(image_diff_dir)
     image = make_test_image()
-    image_diff(image, image)
+    suffix = "".join(random.choices(string.ascii_letters, k=10))
+    image_diff(image, image, suffix=suffix)
 
 
 def test_diff(image_diff):
diff --git a/tests/test_no_throw.py b/tests/test_no_throw.py
new file mode 100644
index 0000000..38ac105
--- /dev/null
+++ b/tests/test_no_throw.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+import os
+import random
+import shutil
+import string
+
+import pytest
+from PIL import Image, ImageDraw
+
+
+def make_test_image(text="Hello world", size=(100, 30)) -> Image:
+    img = Image.new("RGB", size, color=(73, 109, 137))
+
+    d = ImageDraw.Draw(img)
+    d.text((10, 10), text, fill=(255, 255, 0))
+
+    return img
+
+
+@pytest.fixture(scope="session")
+def image_diff_throw_exception() -> bool:
+    """
+    Set default throw exception. By default - True
+    """
+    return False
+
+
+def test_initial_diff(image_diff, image_diff_dir):
+    suffix = "".join(random.choices(string.ascii_letters, k=10))
+    image = make_test_image()
+    assert image_diff(image, image, suffix=suffix)
+
+
+def test_diff(image_diff):
+    image = make_test_image()
+    assert image_diff(image, image)
+
+
+def test_fail_diff(image_diff, image_diff_dir):
+    assert not image_diff(make_test_image("Foo"), make_test_image("Bar"))
+    assert image_diff(make_test_image("Foo"), make_test_image("Bar")) > 0
+
+
+def test_regression(image_regression):
+    image = make_test_image()
+    assert image_regression(image)
+    assert image_regression(image)
+
+
+def test_fail_regression(image_regression):
+    assert image_regression(make_test_image("Foo"))
+    assert not image_regression(make_test_image("Bar"))
+    assert image_regression(make_test_image("Baz")) > 0
+
+
+def test_fail_regression_different_size(image_regression):
+    assert image_regression(make_test_image("Foo"))
+    assert not image_regression(make_test_image("Foo", size=(200, 200)))