Repository: wagtail/wagtail

Issue:
Wagtail 4.2 new incompatibility with dj-inmemorystorage
<!--
Found a bug? Please fill out the sections below. 👍
-->

### Issue Summary

I tried today to upgrade to Wagtail 4.2 and noticed that my image related tests
stopped working. I use [dj-inmemorystorage](https://github.com/waveaccounting/dj-inmemorystorage)
to speed up my file base tests a little bit and it seems like it's incompatible with willow.

### Steps to Reproduce

1. Install `pip install dj-inmemorystorage`
2. Add `DEFAULT_FILE_STORAGE = "inmemorystorage.InMemoryStorage"` to your test settings
3. Running a test creating an Image model will now raise an `willow.image.UnrecognisedImageFormatError` error see below for an example

```python
@pytest.mark.django_db
def test_create_image_from_scratch():
    from django.core.files.uploadedfile import SimpleUploadedFile
    from wagtail.images.models import Image
    # This is a 1x1 black png
    png = (
        b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00"
        b"\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00"
        b"\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc````"
        b"\x00\x00\x00\x05\x00\x01\xa5\xf6E@\x00\x00"
        b"\x00\x00IEND\xaeB`\x82"
    )

    simple_png = SimpleUploadedFile(name="test.png", content=png, content_type="image/png")
    image = Image(file=simple_png)
    image.save()
```

<details>
<summary>Here's the traceback</summary>

```python
╰─>$ pytest -k test_create_image_from_scratch  1283ms  Di  7 Feb 19:21:39 2023
============================= test session starts =============================
platform darwin -- Python 3.11.1, pytest-7.2.1, pluggy-1.0.0
Using --randomly-seed=1234
django: settings: tests.settings (from ini)
rootdir: /Users/jochen/projects/django-cast, configfile: pyproject.toml, testpaths: tests
plugins: Faker-16.6.1, randomly-3.12.0, cov-4.0.0, django-4.5.2, anyio-3.6.2
collected 207 items / 206 deselected / 1 selected

tests/wagtail_image_views_test.py F                                     [100%]

================================== FAILURES ===================================
_______________________ test_create_image_from_scratch ________________________

    @pytest.mark.django_db
    def test_create_image_from_scratch():
        from django.core.files.uploadedfile import SimpleUploadedFile
        from wagtail.images.models import Image
        # This is a 1x1 black png
        png = (
            b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00"
            b"\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00"
            b"\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc````"
            b"\x00\x00\x00\x05\x00\x01\xa5\xf6E@\x00\x00"
            b"\x00\x00IEND\xaeB`\x82"
        )

        simple_png = SimpleUploadedFile(name="test.png", content=png, content_type="image/png")
        image = Image(file=simple_png)
>       image.save()

tests/wagtail_image_views_test.py:19:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/base.py:812: in save
    self.save_base(
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/base.py:863: in save_base
    updated = self._save_table(
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/base.py:1006: in _save_table
    results = self._do_insert(
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/base.py:1047: in _do_insert
    return manager._insert(
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/manager.py:85: in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/query.py:1791: in _insert
    return query.get_compiler(using=using).execute_sql(returning_fields)
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/sql/compiler.py:1659: in execute_sql
    for sql, params in self.as_sql():
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/sql/compiler.py:1583: in as_sql
    value_rows = [
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/sql/compiler.py:1584: in <listcomp>
    [
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/sql/compiler.py:1585: in <listcomp>
    self.prepare_value(field, self.pre_save_val(field, obj))
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/sql/compiler.py:1533: in pre_save_val
    return field.pre_save(obj, add=True)
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/fields/files.py:316: in pre_save
    file.save(file.name, file.file, save=False)
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/fields/files.py:93: in save
    setattr(self.instance, self.field.attname, self.name)
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/fields/files.py:378: in __set__
    self.field.update_dimension_fields(instance, force=True)
../../.virtualenvs/cast/lib/python3.11/site-packages/django/db/models/fields/files.py:490: in update_dimension_fields
    width = file.width
../../.virtualenvs/cast/lib/python3.11/site-packages/django/core/files/images.py:20: in width
    return self._get_image_dimensions()[0]
../../.virtualenvs/cast/lib/python3.11/site-packages/wagtail/images/models.py:201: in _get_image_dimensions
    self._dimensions_cache = self.get_image_dimensions()
../../.virtualenvs/cast/lib/python3.11/site-packages/wagtail/images/models.py:212: in get_image_dimensions
    image = willow.Image.open(self)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

cls = <class 'willow.image.Image'>
f = <WagtailImageFieldFile: original_images/test.png>

    @classmethod
    def open(cls, f):
        # Detect image format
        image_format = imghdr.what(f)

        # Find initial class
        initial_class = INITIAL_IMAGE_CLASSES.get(image_format)
        if not initial_class:
            if image_format:
                raise UnrecognisedImageFormatError("Cannot load %s images" % image_format)
            else:
>               raise UnrecognisedImageFormatError("Unknown image format")
E               willow.image.UnrecognisedImageFormatError: Unknown image format

../../.virtualenvs/cast/lib/python3.11/site-packages/willow/image.py:97: UnrecognisedImageFormatError
=========================== short test summary info ===========================
FAILED tests/wagtail_image_views_test.py::test_create_image_from_scratch - willow.image.UnrecognisedImageFormatError: Unknown image format
====================== 1 failed, 206 deselected in 0.40s ======================
```

</details>

Any other relevant information. For example, why do you consider this a bug and what did you expect to happen instead?

I don't know if this is a bug in Wagtail. The `dj-inmemorystorage` package looks pretty stale maybe it's using
an outdated version of the Django storage API? On the other hand: With Wagtail 4.1.2 everything worked fine.

-   I have confirmed that this issue can be reproduced as described on a fresh Wagtail project: (yes / no)
yes

### Technical details

-   Python version: 3.11.1
-   Django version: 4.1.6
-   Wagtail version: 4.2.0
-   Browser version: no browser :)



Discussion:
@ephes thank you for this report. Is there any chance you can update the issue with a traceback? We changed a few bits around image handling in #8974 (and #9572) in preparation for SVG support
Can confirm that I'm seeing the same test failure, and 912747f6aeb585a688b9d3536cf7bd7e1c94487c is the first commit where it happens.
This is proving a tricky one to track down... there are two calls being made to `get_image_dimensions`, one during the `Image(file=simple_png)` and the other during `image.save()`. On the second one, reading the file with `self.read(...)` only returns the last few bytes, even if we do `self.seek(0)` first.

Adding a `content.seek(0)` to Django's [FieldFile.save](https://github.com/django/django/blob/2fd755b361d3da2cd0440fc9839feb2bb69b027b/django/db/models/fields/files.py#L91) fixes the problem, but there doesn't seem to be any place to add a `seek` within the Willow code that does the job... I suspect that by the time it reaches the second `get_image_dimensions` call, it's already saved a truncated copy of the file.

My best guess is that dj-inmemorystorage is failing to do a `seek(0)` on `open` [as it's supposed to](https://docs.djangoproject.com/en/4.1/ref/files/file/#django.core.files.File.open), but I need to do more digging to confirm exactly where it's going wrong.
Further digging:

[The docstring for Storage.save](https://docs.djangoproject.com/en/4.1/_modules/django/core/files/storage/#Storage.save) says

> The content should be a proper File object or any Python file-like object, ready to be read from the beginning.

which is slightly ambiguous about whether it means "it should be a file object that we can call `seek(0)` on" or "it should be a file object that `seek(0)` has _already_ been called on". Either way, at this point in the code the file pointer is _not_ at the start (it's 66 bytes in, to be exact).

When using the default storage, this doesn't matter because [it goes through `content.chunks()`](https://github.com/django/django/blob/590a92e4562c8ebd3eb25dc1e878bd1e3eb758d4/django/core/files/storage.py#L318), which does its own `seek(0)`. However, [dj-inmemorystorage just performs `content.read()`](https://github.com/waveaccounting/dj-inmemorystorage/blob/f01bba32dc6937e6a964c65ae05147e5f893f8e6/inmemorystorage/storage.py#L194).

It would be instructive to see what other storage backends such as django-storages do here - if they also expect the file pointer to be at the start here, then this could be a nastier bug than anticipated (especially as it's liable to only surface in production).

I'm now seeing that adding `image.file.seek(0)` before the `image.save()` in the test does fix this - I thought I tried that before with no success... - so the fix is presumably to ensure that `Image(file=simple_png)` returns with the file pointer reset.
> It would be instructive to see what other storage backends such as django-storages do here

Looks like the django-storages backends are consistent about calling `seek`, `open` or equivalent, so that's good!

Resolving patch:
diff --git a/wagtail/images/models.py b/wagtail/images/models.py
--- a/wagtail/images/models.py
+++ b/wagtail/images/models.py
@@ -214,6 +214,8 @@ def get_image_dimensions(self):
         finally:
             if close:
                 self.close()
+            else:
+                self.seek(0)
 
 
 class WagtailImageField(models.ImageField):


Test patch:
diff --git a/wagtail/test/dummy_external_storage.py b/wagtail/test/dummy_external_storage.py
--- a/wagtail/test/dummy_external_storage.py
+++ b/wagtail/test/dummy_external_storage.py
@@ -5,6 +5,7 @@
 #  - Calling .path on the storage or image file raises NotImplementedError
 #  - File.open() after the file has been closed raises an error
 #  - File.size exceptions raise DummyExternalStorageError
+#  - Storage._save() fails loudly if the content file's pointer is not at the start
 
 from django.core.files.base import File
 from django.core.files.storage import FileSystemStorage, Storage
@@ -26,11 +27,16 @@ def _open(self, name, mode="rb"):
         # File object
         return DummyExternalStorageFile(open(self.wrapped.path(name), mode))
 
-    # Wrap all other functions
-
     def _save(self, name, content):
+        file_pos = content.tell()
+        if file_pos != 0:
+            raise ValueError(
+                "Content file pointer should be at 0 - got %d instead" % file_pos
+            )
         return self.wrapped._save(name, content)
 
+    # Wrap all other functions
+
     def delete(self, name):
         self.wrapped.delete(name)
 
diff --git a/wagtail/tests/test_tests.py b/wagtail/tests/test_tests.py
--- a/wagtail/tests/test_tests.py
+++ b/wagtail/tests/test_tests.py
@@ -1,9 +1,11 @@
 import json
 
+from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase
 
 from wagtail.admin.tests.test_contentstate import content_state_equal
 from wagtail.models import PAGE_MODEL_CLASSES, Page, Site
+from wagtail.test.dummy_external_storage import DummyExternalStorage
 from wagtail.test.testapp.models import (
     BusinessChild,
     BusinessIndex,
@@ -429,3 +431,28 @@ def test_rich_text_with_custom_features(self):
     def test_rich_text_with_alternative_editor(self):
         result = rich_text("<h2>title</h2><p>para</p>", editor="custom")
         self.assertEqual(result, "<h2>title</h2><p>para</p>")
+
+
+class TestDummyExternalStorage(WagtailTestUtils, TestCase):
+    def test_save_with_incorrect_file_object_position(self):
+        """
+        Test that DummyExternalStorage correctly warns about attempts
+        to write files that are not rewound to the start
+        """
+        # This is a 1x1 black png
+        png = (
+            b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00"
+            b"\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00"
+            b"\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc````"
+            b"\x00\x00\x00\x05\x00\x01\xa5\xf6E@\x00\x00"
+            b"\x00\x00IEND\xaeB`\x82"
+        )
+        simple_png = SimpleUploadedFile(
+            name="test.png", content=png, content_type="image/png"
+        )
+        simple_png.read()
+        with self.assertRaisesMessage(
+            ValueError,
+            "Content file pointer should be at 0 - got 70 instead",
+        ):
+            DummyExternalStorage().save("test.png", simple_png)