Repository: svthalia/concrexit

Issue:
Use base64 images in newsletters archive
### Describe the change
Apply a small transformation to the newsletter HTML upon saving it to disk.
As described in  https://gist.github.com/pansapiens/110431456e8a4ba4f2eb.


### Motivation
We both want to be able to send newsletters with dynamic partner images, and we would like to keep a proper archive of newsletters that contain all media. Including images.

### Current implementation
We save the thumbnail url to the HTML file that is archived.

### Suggested implementation
Convert all images in the HTML to base64.

### Additional context
This may need a migration to apply the transformation to all existing archive files.



Discussion:

Resolving patch:
diff --git a/website/newsletters/migrations/0015_embed_images_html.py b/website/newsletters/migrations/0015_embed_images_html.py
new file mode 100644
--- /dev/null
+++ b/website/newsletters/migrations/0015_embed_images_html.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import os
+
+from django.conf import settings
+from django.db import migrations
+
+from newsletters import services
+
+
+def embed_images_html(_, __):
+    cache_dir = os.path.join(settings.MEDIA_ROOT, "newsletters")
+    if not os.path.isdir(cache_dir):
+        return
+
+    for file in os.listdir(cache_dir):
+        with open(os.path.join(cache_dir, file), "r+", encoding="utf-8") as newsletter_file:
+            contents = newsletter_file.read()
+            new_content = services.embed_linked_html_images(contents)
+            newsletter_file.write(new_content)
+
+
+class Migration(migrations.Migration):
+    atomic = False
+    dependencies = [
+        ('newsletters', '0014_auto_20211216_2257'),
+    ]
+
+    operations = [
+        migrations.RunPython(embed_images_html, lambda *args: None)
+    ]
diff --git a/website/newsletters/services.py b/website/newsletters/services.py
--- a/website/newsletters/services.py
+++ b/website/newsletters/services.py
@@ -1,5 +1,11 @@
+import base64
+import logging
 import os
+from io import BytesIO
 
+import requests
+from PIL import Image
+from bs4 import BeautifulSoup
 from django.conf import settings
 from django.core.files.base import ContentFile
 from django.core.files.storage import DefaultStorage
@@ -12,6 +18,8 @@
 from partners.models import Partner
 from pushnotifications.models import Message, Category
 
+logger = logging.getLogger(__name__)
+
 
 def write_to_file(pk, lang, html_message):
     """Write newsletter to a file."""
@@ -45,10 +53,33 @@ def save_to_disk(newsletter):
         }
 
         html_message = html_template.render(context)
+        html_message = embed_linked_html_images(html_message)
 
         write_to_file(newsletter.pk, language[0], html_message)
 
 
+def embed_linked_html_images(html_input):
+    bs = BeautifulSoup(html_input, "html.parser")
+
+    output = html_input
+    images = bs.findAll("img")
+
+    for image in images:
+        try:
+            source = image["src"]
+            response = requests.get(source)
+            image = Image.open(BytesIO(response.content))
+            buffer = BytesIO()
+            image.save(buffer, format="png")
+            base64_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
+            encoded_image = "data:image/png;base64, " + base64_image
+            output = output.replace(source, encoded_image)
+        except IOError:
+            logger.warning(f"Image could not be found: {image}")
+
+    return output
+
+
 def get_agenda(start_date):
     end_date = start_date + timezone.timedelta(weeks=2)
     published_events = Event.objects.filter(published=True)


Test patch: