Repository: hydroshare/hydroshare

Issue:
Carriage returns in abstract prevent save due to XML parsing
Related to
https://github.com/hydroshare/hydroshare/issues/5306


Discussion:

Resolving patch:
diff --git a/hs_core/forms.py b/hs_core/forms.py
--- a/hs_core/forms.py
+++ b/hs_core/forms.py
@@ -10,7 +10,7 @@
 from crispy_forms.bootstrap import Field
 
 from .models import Party, Creator, Contributor, validate_hydroshare_user_id, Relation, Identifier, \
-    FundingAgency, Description, validate_abstract
+    FundingAgency, Description, validate_abstract, clean_abstract as clean_abstract_core
 
 
 class Helper(object):
@@ -735,6 +735,11 @@ class AbstractValidationForm(forms.Form):
 
     abstract = forms.CharField(max_length=5000, validators=[validate_abstract])
 
+    def clean_abstract(self):
+        """Ensure that the abstract doesn't contain invalid characters."""
+        abstract = self.cleaned_data['abstract']
+        return clean_abstract_core(abstract)
+
 
 class RightsValidationForm(forms.Form):
     """Validate Rights form with statement and URL field."""
diff --git a/hs_core/models.py b/hs_core/models.py
--- a/hs_core/models.py
+++ b/hs_core/models.py
@@ -1,10 +1,10 @@
 """Declare critical models for Hydroshare hs_core app."""
 import copy
-import difflib
 import json
 import logging
 import os.path
 import re
+import sys
 import unicodedata
 import urllib.parse
 from uuid import uuid4
@@ -55,6 +55,46 @@
 from .languages_iso import languages as iso_languages
 
 
+def clean_abstract(original_string):
+    """Clean abstract for XML inclusion.
+
+    This function takes an original string and removes any illegal XML characters
+    from it. It uses regular expressions to identify and remove the illegal characters.
+
+    Args:
+        original_string (str): The original string to be cleaned.
+
+    Returns:
+        str: The cleaned string with illegal XML characters removed.
+
+    Raises:
+        ValidationError: If there is an error cleaning the abstract.
+
+    """
+    # https://stackoverflow.com/a/64570125
+    try:
+        illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x1F),
+                           (0x7F, 0x84), (0x86, 0x9F),
+                           (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF)]
+        if sys.maxunicode >= 0x10000:  # not narrow build
+            illegal_unichrs.extend([(0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF),
+                                    (0x3FFFE, 0x3FFFF), (0x4FFFE, 0x4FFFF),
+                                    (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF),
+                                    (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF),
+                                    (0x9FFFE, 0x9FFFF), (0xAFFFE, 0xAFFFF),
+                                    (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF),
+                                    (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF),
+                                    (0xFFFFE, 0xFFFFF), (0x10FFFE, 0x10FFFF)])
+
+        illegal_ranges = [fr'{chr(low)}-{chr(high)}' for (low, high) in illegal_unichrs]
+        xml_illegal_character_regex = '[' + ''.join(illegal_ranges) + ']'
+        illegal_xml_chars_re = re.compile(xml_illegal_character_regex)
+        filtered_string = illegal_xml_chars_re.sub('', original_string)
+        return filtered_string
+    except (KeyError, TypeError) as ex:
+        raise ValidationError(f"Error cleaning abstract: {ex}")
+
+
 def clean_for_xml(s):
     """
     Remove all control characters from a unicode string in preparation for XML inclusion
@@ -65,6 +105,7 @@ def clean_for_xml(s):
     * Space-pad paragraph and NL symbols as necessary
 
     """
+    # https://www.w3.org/TR/REC-xml/#sec-line-ends
     CR = chr(0x23CE)  # carriage return unicode SYMBOL
     PARA = chr(0xB6)  # paragraph mark unicode SYMBOL
     output = ''
@@ -747,16 +788,27 @@ class Meta:
 
 
 def validate_abstract(value):
-    """Validate that an abstract is valid."""
-    err_message = 'The abstract is not valid. It contains characters that are not XML compatible.'
+    """
+    Validates the abstract value by ensuring it can serialize as XML.
+
+    Args:
+        value (str): The abstract value to be validated.
+
+    Raises:
+        ValidationError: If there is an error parsing the abstract as XML.
+
+    Returns:
+        None
+    """
+    err_message = 'Error parsing abstract as XML.'
     if value:
         try:
-            clean = clean_for_xml(value)
-            assert (len(clean) == len(value))
-            match_ratio = difflib.SequenceMatcher(None, clean.splitlines(), value.splitlines()).ratio()
-            assert (match_ratio == 1.0)
-        except AssertionError:
-            raise ValidationError(err_message)
+            ROOT = etree.Element('root')
+            body_node = etree.SubElement(ROOT, 'body')
+            c_abstract = clean_abstract(value)
+            etree.SubElement(body_node, 'description').text = c_abstract
+        except Exception as ex:
+            raise ValidationError(f'{err_message}, more info: {ex}')
 
 
 @rdf_terms(DC.description, abstract=DCTERMS.abstract)
@@ -778,13 +830,13 @@ class Meta:
     @classmethod
     def create(cls, **kwargs):
         """Define custom update method for Description model."""
-        validate_abstract(kwargs['abstract'])
+        kwargs['abstract'] = clean_abstract(kwargs['abstract'])
         return super(Description, cls).create(**kwargs)
 
     @classmethod
     def update(cls, element_id, **kwargs):
         """Define custom update method for Description model."""
-        validate_abstract(kwargs['abstract'])
+        kwargs['abstract'] = clean_abstract(kwargs['abstract'])
         super(Description, cls).update(element_id, **kwargs)
 
     @classmethod
@@ -4104,7 +4156,8 @@ def create_date_node(date, date_type):
         # create update_date sub element
         create_date_node(date=self.updated, date_type="update_date")
         # create dataset description sub element
-        etree.SubElement(dataset_node, 'description').text = self.metadata.description.abstract
+        c_abstract = clean_abstract(self.metadata.description.abstract)
+        etree.SubElement(dataset_node, 'description').text = c_abstract
         # funder related elements
         funders = self.metadata.funding_agencies.all()
         if funders:


Test patch:
diff --git a/hs_core/tests/api/native/test_clean_abstract.py b/hs_core/tests/api/native/test_clean_abstract.py
new file mode 100644
--- /dev/null
+++ b/hs_core/tests/api/native/test_clean_abstract.py
@@ -0,0 +1,81 @@
+from django.test import TestCase
+from hs_core.models import clean_abstract
+
+
+class TestCleaningAbstract(TestCase):
+    def test_no_convert_newlines(self):
+        self.assertEqual(clean_abstract("it \r\n \r\n is \r\n \r\n finished"),
+                         "it \r\n \r\n is \r\n \r\n finished")
+
+    def test_convert_u0003(self):
+        self.assertEqual(clean_abstract("it \n\u0003\n is \n\u0003\n finished"),
+                         "it \n\n is \n\n finished")
+
+    def test_convert_u0003_2(self):
+        """ embedded control characters in words are converted to null """
+        self.assertEqual(clean_abstract("it \n \n i\u0003s \n \n finished"),
+                         "it \n \n is \n \n finished")
+
+    def test_convert_newlines_nonsense(self):
+        """ complete nonsense is converted to null """
+        self.assertEqual(clean_abstract("it \n\u0003\u0004 \n i\u0003\u0006\u0001s \n \n finished"),
+                         "it \n \n is \n \n finished")
+
+    def test_convert_newlines_arb(self):
+        """ arbitrary numbers of newlines not modified """
+        self.assertEqual(clean_abstract("it \n\n\r\n\r\n is \n\n\n\n finished"),
+                         "it \n\n\r\n\r\n is \n\n\n\n finished")
+
+    def test_clean_abstract_no_illegal_chars(self):
+        """ No illegal characters in the abstract """
+        original_string = "This is a valid abstract."
+        expected_result = "This is a valid abstract."
+        self.assertEqual(clean_abstract(original_string), expected_result)
+
+    def test_clean_abstract_with_illegal_chars(self):
+        """ Abstract contains illegal characters """
+        original_string = "This is an abstract with \x00 illegal \x1F characters."
+        expected_result = "This is an abstract with  illegal  characters."
+        self.assertEqual(clean_abstract(original_string), expected_result)
+
+    def test_clean_abstract_empty_string(self):
+        """ Empty abstract """
+        original_string = ""
+        expected_result = ""
+        self.assertEqual(clean_abstract(original_string), expected_result)
+
+    def test_clean_abstract_with_unicode_chars(self):
+        """ Abstract contains unicode characters """
+        original_string = "This is an abstract with 𝓊𝓃𝒾𝒸𝑜𝒹𝑒 characters."
+        expected_result = "This is an abstract with 𝓊𝓃𝒾𝒸𝑜𝒹𝑒 characters."
+        self.assertEqual(clean_abstract(original_string), expected_result)
+
+    def test_clean_abstract_with_ascii_chars(self):
+        """ Abstract contains ASCII characters """
+        original_string = f"This is an abstract with {chr(0x23CE)} characters."
+        expected_result = f"This is an abstract with {chr(0x23CE)} characters."
+        self.assertEqual(clean_abstract(original_string), expected_result)
+
+    def test_clean_abstract_with_html_tags(self):
+        """ Abstract contains HTML tags """
+        original_string = "This is an abstract with <b>HTML</b> tags."
+        expected_result = "This is an abstract with <b>HTML</b> tags."
+        self.assertEqual(clean_abstract(original_string), expected_result)
+
+    def test_clean_abstract_with_html_entities(self):
+        """ Abstract contains HTML entities """
+        original_string = "This is an abstract with &lt;HTML&gt; entities."
+        expected_result = "This is an abstract with &lt;HTML&gt; entities."
+        self.assertEqual(clean_abstract(original_string), expected_result)
+
+    def test_clean_abstract_with_html_entities_and_tags(self):
+        """ Abstract contains HTML entities and tags """
+        original_string = "This is an abstract with &lt;b&gt;HTML&lt;/b&gt; entities and tags."
+        expected_result = "This is an abstract with &lt;b&gt;HTML&lt;/b&gt; entities and tags."
+        self.assertEqual(clean_abstract(original_string), expected_result)
+
+    def test_clean_abstract_with_markdown(self):
+        """ Abstract contains Markdown """
+        original_string = "This is an abstract with **Markdown**."
+        expected_result = "This is an abstract with **Markdown**."
+        self.assertEqual(clean_abstract(original_string), expected_result)
diff --git a/hs_core/tests/api/native/test_core_metadata.py b/hs_core/tests/api/native/test_core_metadata.py
--- a/hs_core/tests/api/native/test_core_metadata.py
+++ b/hs_core/tests/api/native/test_core_metadata.py
@@ -1402,40 +1402,45 @@ def test_description(self):
         )
 
     def test_description_xml(self):
-
+        """
+        Test case for checking that illegal characters are cleaned during abstract create/update
+        """
         # test that the resource metadata does not contain abstract
         self.assertEqual(
             self.res.metadata.description, None, msg="Abstract exists for the resource"
         )
 
-        # test adding invalid XML raises exception
-        self.assertRaises(
-            ValidationError,
-            lambda: resource.create_metadata_element(
-                self.res.short_id,
-                "description",
-                abstract="analysis throughput rates greater than 7 h1. The soil dept",
-            ),
+        # test adding invalid XML doesn't raise exception
+        resource.create_metadata_element(
+            self.res.short_id,
+            "description",
+            abstract="greater than 7 h1. The soil dept",
         )
 
-        # create a abstract for the resource
-        resource.create_metadata_element(
-            self.res.short_id, "description", abstract="new abstract for the resource"
+        self.assertEqual(
+            self.res.metadata.description.abstract, "greater than 7 h1. The soil dept"
         )
 
-        # attempt update, this should fail
-        self.assertRaises(
-            ValidationError,
-            lambda: resource.update_metadata_element(
-                self.res.short_id,
-                "description",
-                self.res.metadata.description.id,
-                abstract="analysis throughput rates greater than 7 h1. The soil dept",
-            )
+        # update
+        resource.update_metadata_element(
+            self.res.short_id,
+            "description",
+            self.res.metadata.description.id,
+            abstract="new than 7 h\x1F1. The soil dept",
+        )
+        self.assertEqual(
+            self.res.metadata.description.abstract, "new than 7 h1. The soil dept"
         )
-        # the abstract should remain unchanged
+
+        resource.update_metadata_element(
+            self.res.short_id,
+            "description",
+            self.res.metadata.description.id,
+            abstract="newer than 7 h\r\r1. The soil dept",
+        )
+
         self.assertEqual(
-            self.res.metadata.description.abstract, "new abstract for the resource"
+            self.res.metadata.description.abstract, "newer than 7 h\r\r1. The soil dept"
         )
 
     def test_format(self):
diff --git a/hs_core/tests/api/native/test_publish_resource.py b/hs_core/tests/api/native/test_publish_resource.py
--- a/hs_core/tests/api/native/test_publish_resource.py
+++ b/hs_core/tests/api/native/test_publish_resource.py
@@ -235,51 +235,15 @@ def test_last_updated(self):
     def test_crossref_deposit_xml(self):
         """Test that the crossref deposit xml is generated correctly for a resource"""
 
-        # create abstract metadata element
-        self.res.metadata.create_element('description', abstract='This is a test abstract')
-        # add an organization as an author
-        self.res.metadata.create_element('creator', organization='Utah State University')
-        # add a person with ORCID identifier as an author
-        identifiers = {"ORCID": 'https://orcid.org/0000-0002-1825-0097'}
-        self.res.metadata.create_element('creator', name='John Smith', identifiers=identifiers)
-        # add a funder that can be found in crossref funders registry
-        self.res.metadata.create_element('fundingagency', agency_name='National Science Foundation',
-                                         award_title='NSF Award', award_number='12345', agency_url='https://nsf.gov')
-        # add a funder that can't be found in crossref funders registry
-        self.res.metadata.create_element('fundingagency', agency_name='Utah Water Research Laboratory')
+        self.create_xml_test_resource()
+        self.freeze_and_assert()
 
-        if not hasattr(settings, 'DEFAULT_SUPPORT_EMAIL'):
-            settings.DEFAULT_SUPPORT_EMAIL = "help@cuahsi.org"
-
-        freeze = arrow.now().format("YYYY-MM-DD HH:mm:ss")
-        freezer = freeze_time(freeze)
-        freezer.start()
-
-        # set doi - simulating published resource
-        self.res.doi = get_resource_doi(self.res.short_id)
-        self.res.save()
-        crossref_xml = self.res.get_crossref_deposit_xml()
-        crossref_xml = crossref_xml.strip()
-        timestamp = arrow.now().format("YYYYMMDDHHmmss")
-        res_id = self.res.short_id
-        created_date = self.res.created
-        updated_date = self.res.updated
-        # use updated date as published date as we don't have a published date for this test resource
-        # also when we do real publication the publication date is the same as updated date at the time of publication
-        published_date = updated_date
-        site_url = hydroshare.utils.current_site_url()
-        support_email = settings.DEFAULT_SUPPORT_EMAIL
-        expected_xml = self._get_expected_crossref_xml(res_id=res_id, timestamp=timestamp,
-                                                       created_date=created_date, updated_date=updated_date,
-                                                       published_date=published_date, site_url=site_url,
-                                                       support_email=support_email)
-        expected_xml = expected_xml.strip()
-
-        freezer.stop()
-
-        self.assertTrue(len(crossref_xml) == len(expected_xml))
-        match_ratio = difflib.SequenceMatcher(None, crossref_xml.splitlines(), expected_xml.splitlines()).ratio()
-        self.assertTrue(match_ratio == 1.0, msg="crossref xml is not as expected")
+    def test_crossref_deposit_dirty_xml(self):
+        """Test that the crossref deposit xml is generated correctly for a resource with dirty abstract"""
+        self.create_xml_test_resource()
+        self.res.metadata.update_element('description', self.res.metadata.description.id,
+                                         abstract='This is a test abstract\x1F')
+        self.freeze_and_assert()
 
     @staticmethod
     def _get_expected_crossref_xml(res_id, timestamp, created_date, updated_date, published_date, site_url,
@@ -360,3 +324,51 @@ def _get_expected_crossref_xml(res_id, timestamp, created_date, updated_date, pu
   </body>
 </doi_batch>""" # noqa
         return expected_xml
+
+    def create_xml_test_resource(self):
+        # create abstract metadata element
+        self.res.metadata.create_element('description', abstract='This is a test abstract')
+        # add an organization as an author
+        self.res.metadata.create_element('creator', organization='Utah State University')
+        # add a person with ORCID identifier as an author
+        identifiers = {"ORCID": 'https://orcid.org/0000-0002-1825-0097'}
+        self.res.metadata.create_element('creator', name='John Smith', identifiers=identifiers)
+        # add a funder that can be found in crossref funders registry
+        self.res.metadata.create_element('fundingagency', agency_name='National Science Foundation',
+                                         award_title='NSF Award', award_number='12345', agency_url='https://nsf.gov')
+        # add a funder that can't be found in crossref funders registry
+        self.res.metadata.create_element('fundingagency', agency_name='Utah Water Research Laboratory')
+
+        if not hasattr(settings, 'DEFAULT_SUPPORT_EMAIL'):
+            settings.DEFAULT_SUPPORT_EMAIL = "help@cuahsi.org"
+
+    def freeze_and_assert(self):
+        freeze = arrow.now().format("YYYY-MM-DD HH:mm:ss")
+        freezer = freeze_time(freeze)
+        freezer.start()
+
+        # set doi - simulating published resource
+        self.res.doi = get_resource_doi(self.res.short_id)
+        self.res.save()
+        crossref_xml = self.res.get_crossref_deposit_xml()
+        crossref_xml = crossref_xml.strip()
+        timestamp = arrow.now().format("YYYYMMDDHHmmss")
+        res_id = self.res.short_id
+        created_date = self.res.created
+        updated_date = self.res.updated
+        # use updated date as published date as we don't have a published date for this test resource
+        # also when we do real publication the publication date is the same as updated date at the time of publication
+        published_date = updated_date
+        site_url = hydroshare.utils.current_site_url()
+        support_email = settings.DEFAULT_SUPPORT_EMAIL
+        expected_xml = self._get_expected_crossref_xml(res_id=res_id, timestamp=timestamp,
+                                                       created_date=created_date, updated_date=updated_date,
+                                                       published_date=published_date, site_url=site_url,
+                                                       support_email=support_email)
+        expected_xml = expected_xml.strip()
+
+        freezer.stop()
+
+        self.assertTrue(len(crossref_xml) == len(expected_xml))
+        match_ratio = difflib.SequenceMatcher(None, crossref_xml.splitlines(), expected_xml.splitlines()).ratio()
+        self.assertTrue(match_ratio == 1.0, msg="crossref xml is not as expected")