Repository: ietf-tools/datatracker

Issue:
unicode validation and/or presentation error
Somehow we have a set of minutes that cannot be inspected at
/meeting/interim-2021-lpwan-02/materials/minutes-interim-2021-lpwan-02-202101191600-00.md
but can at
/doc/minutes-interim-2021-lpwan-02-202101191600/00/

We should find what codepath let us get into that state.

Interestingly https://datatracker.ietf.org/meeting/interim-2021-lpwan-02/materials/minutes-interim-2021-lpwan-02-202101191600-02.md (and I assume -01) do not crash, so Pascal probably saw and worked around the issue during submission.



Discussion:

Resolving patch:
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -261,7 +261,7 @@ def materials_document(request, document, num=None, ext=None):
                     content_type = content_type.replace('plain', 'markdown', 1)
                     break;
                 elif atype[0] == 'text/html':
-                    bytes = "<html>\n<head><base target=\"_blank\" /></head>\n<body>\n%s\n</body>\n</html>\n" % markdown.markdown(bytes.decode())
+                    bytes = "<html>\n<head><base target=\"_blank\" /></head>\n<body>\n%s\n</body>\n</html>\n" % markdown.markdown(bytes.decode(encoding=chset))
                     content_type = content_type.replace('plain', 'html', 1)
                     break;
                 elif atype[0] == 'text/plain':


Test patch:
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -121,7 +121,7 @@ def tearDown(self):
         settings.MEETINGHOST_LOGO_PATH = self.saved_meetinghost_logo_path
         super().tearDown()
 
-    def write_materials_file(self, meeting, doc, content):
+    def write_materials_file(self, meeting, doc, content, charset="utf-8"):
         path = os.path.join(self.materials_dir, "%s/%s/%s" % (meeting.number, doc.type_id, doc.uploaded_filename))
 
         dirname = os.path.dirname(path)
@@ -129,7 +129,7 @@ def write_materials_file(self, meeting, doc, content):
             os.makedirs(dirname)
 
         if isinstance(content, str):
-            content = content.encode()
+            content = content.encode(charset)
         with io.open(path, "wb") as f:
             f.write(content)
 
@@ -362,6 +362,32 @@ def test_interim_materials(self):
 
         self.do_test_materials(meeting, session)
 
+    @override_settings(MEETING_MATERIALS_SERVE_LOCALLY=True)
+    def test_meeting_materials_non_utf8(self):
+        meeting = make_meeting_test_data()
+        session = Session.objects.filter(meeting=meeting, group__acronym="mars").first()
+        doc = session.materials.get(type="minutes")
+        self.write_materials_file(meeting,
+                                  doc,
+                                  "1. More work items underway\n\n2. The draft will be finished before next meeting\n\n - É",
+                                  charset="iso-8859-1")
+        url = urlreverse("ietf.meeting.views.materials_document",
+                         kwargs=dict(num=meeting.number, document=session.minutes()))
+
+        for accept, cont_type, content in [
+                ('text/html,text/plain,text/markdown',  'text/html',     '<li>\n<p>More work items underway</p>\n</li>'),
+                ('text/markdown,text/html,text/plain',  'text/markdown', '1. More work items underway'),
+                ('text/plain,text/markdown, text/html', 'text/plain',    '1. More work items underway'),
+                ('text/html',                           'text/html',     '<li>\n<p>More work items underway</p>\n</li>'),
+                ('text/markdown',                       'text/markdown', '1. More work items underway'),
+                ('text/plain',                          'text/plain',    '1. More work items underway'),
+            ]:
+            client = Client(HTTP_ACCEPT=accept)
+            r = client.get(url)
+            rtype = r['Content-Type'].split(';')[0]
+            self.assertEqual(cont_type, rtype)
+            self.assertContains(r, content)
+
     @override_settings(MEETING_MATERIALS_SERVE_LOCALLY=True)
     def do_test_materials(self, meeting, session):