Repository: nextcloud/appstore

Issue:
rewrite info.xml on upload to match required schema order



Discussion:
Note that apps are cryptographically signed, you can rewrite it for processing but not for delivery to the end-user 😉 

the rewrite is only needed during upload to the AppStore. Currently if the xml in the tarball isn't in the order the xsd requires the validation fails, even if the info.xml contains all the required information.
So we need an extra transformation to get the info.xml in the required order and validate with the schema.
As @BernhardPosselt said xml f*ckery. :)


Resolving patch:
diff --git a/nextcloudappstore/core/api/v1/release/__init__.py b/nextcloudappstore/core/api/v1/release/__init__.py
--- a/nextcloudappstore/core/api/v1/release/__init__.py
+++ b/nextcloudappstore/core/api/v1/release/__init__.py
@@ -11,3 +11,4 @@ def __init__(self) -> None:
         self.download_max_size = 20 * (1024 ** 2)
         self.info_schema = read_relative_file(__file__, 'info.xsd')
         self.info_xslt = read_relative_file(__file__, 'info.xslt')
+        self.pre_info_xslt = read_relative_file(__file__, 'pre-info.xslt')
diff --git a/nextcloudappstore/core/api/v1/release/parser.py b/nextcloudappstore/core/api/v1/release/parser.py
--- a/nextcloudappstore/core/api/v1/release/parser.py
+++ b/nextcloudappstore/core/api/v1/release/parser.py
@@ -23,6 +23,10 @@ class InvalidAppPackageStructureException(APIException):
     pass
 
 
+class XMLSyntaxError(APIException):
+    pass
+
+
 class GunZipAppMetadataExtractor:
     def __init__(self, config: ReleaseConfig) -> None:
         """
@@ -96,29 +100,39 @@ def element_to_dict(element: Any) -> Dict:
         return {key: element.text}
 
 
-def parse_app_metadata(xml: str, schema: str, xslt: str) -> Dict:
+def parse_app_metadata(xml: str, schema: str, pre_xslt: str,
+                       xslt: str) -> Dict:
     """
     Parses, validates and maps the xml onto a dict
     :argument xml the info.xml string to parse
     :argument schema the schema xml as string
+    :argument pre_xslt xslt which is run before validation to ensure that
+    everything is in the correct order and that unknown elements are excluded
     :argument xslt the xslt to transform it to a matching structure
     :raises InvalidAppMetadataXmlException if the schema does not validate
     :return the parsed xml as dict
     """
     parser = lxml.etree.XMLParser(resolve_entities=False, no_network=True,
                                   remove_comments=True, load_dtd=False,
-                                  remove_blank_text=True, dtd_validation=False)
-    schema_doc = lxml.etree.fromstring(bytes(schema, encoding='utf-8'), parser)
-    doc = lxml.etree.fromstring(bytes(xml, encoding='utf-8'), parser)
+                                  remove_blank_text=True, dtd_validation=False
+                                  )
+    try:
+        doc = lxml.etree.fromstring(bytes(xml, encoding='utf-8'), parser)
+    except lxml.etree.XMLSyntaxError as e:
+        msg = 'info.xml contains malformed xml: %s' % e
+        raise XMLSyntaxError(msg)
     for _ in doc.iter(lxml.etree.Entity):
         raise InvalidAppMetadataXmlException('Must not contain entities')
+    pre_transform = lxml.etree.XSLT(lxml.etree.XML(pre_xslt))
+    pre_transformed_doc = pre_transform(doc)
+    schema_doc = lxml.etree.fromstring(bytes(schema, encoding='utf-8'), parser)
     schema = lxml.etree.XMLSchema(schema_doc)
     try:
-        schema.assertValid(doc)  # type: ignore
+        schema.assertValid(pre_transformed_doc)  # type: ignore
     except lxml.etree.DocumentInvalid as e:
         msg = 'info.xml did not validate: %s' % e
         raise InvalidAppMetadataXmlException(msg)
     transform = lxml.etree.XSLT(lxml.etree.XML(xslt))
-    transformed_doc = transform(doc)
+    transformed_doc = transform(pre_transformed_doc)
     mapped = element_to_dict(transformed_doc.getroot())
     return mapped
diff --git a/nextcloudappstore/core/api/v1/release/provider.py b/nextcloudappstore/core/api/v1/release/provider.py
--- a/nextcloudappstore/core/api/v1/release/provider.py
+++ b/nextcloudappstore/core/api/v1/release/provider.py
@@ -22,6 +22,7 @@ def get_release_info(self, url: str) -> Dict:
         ) as download:
             xml = self.extractor.extract_app_metadata(download.filename)
             info = parse_app_metadata(xml, self.config.info_schema,
+                                      self.config.pre_info_xslt,
                                       self.config.info_xslt)
             # generate sha256sum for archive
             with open(download.filename, 'rb') as f:


Test patch:
diff --git a/nextcloudappstore/core/api/v1/tests/data/infoxmls/broken-xml.xml b/nextcloudappstore/core/api/v1/tests/data/infoxmls/broken-xml.xml
new file mode 100644
--- /dev/null
+++ b/nextcloudappstore/core/api/v1/tests/data/infoxmls/broken-xml.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<info xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../../appmetadata/info.xsd">
+    <id>news</id>
+    <name>News</name>
+    <description>An RSS/Atom feed reader</description>
+    <version>8.8.2</version>
+    <licence>agpl</licence>
+    <author>Bernhard Posselt</author>
+    <category>multimedia</category>
+    <dependencies
+        <owncloud min-version="9.0"/>
+    </dependencies>
+</info>
diff --git a/nextcloudappstore/core/api/v1/tests/data/infoxmls/news.xml b/nextcloudappstore/core/api/v1/tests/data/infoxmls/news.xml
new file mode 100644
--- /dev/null
+++ b/nextcloudappstore/core/api/v1/tests/data/infoxmls/news.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+<info  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../../release/info.xsd">
+    <!-- metadata -->
+    <id>news</id>
+    <name>News</name>
+    <description>An RSS/Atom feed reader</description>
+    <author>Bernhard Posselt, Alessandro Cosentino, Jan-Christoph Borchardt</author>
+    <category>multimedia</category>
+    <licence>agpl</licence>
+    <version>8.8.2</version>
+    <namespace>News</namespace>
+
+    <!-- resources -->
+    <documentation>
+        <user>https://github.com/owncloud/news/wiki#user-documentation</user>
+        <admin>https://github.com/owncloud/news#readme</admin>
+        <developer>https://github.com/owncloud/news/wiki#developer-documentation</developer>
+    </documentation>
+    <website>https://github.com/owncloud/news</website>
+    <bugs>https://github.com/owncloud/news/issues</bugs>
+    <repository type="git">https://github.com/owncloud/news.git</repository>
+
+    <!-- dependencies -->
+    <dependencies>
+        <owncloud min-version="9.0" max-version="9.1"/>
+        <php min-version="5.6" min-int-size="64"/>
+        <database min-version="9.4">pgsql</database>
+        <database>sqlite</database>
+        <database min-version="5.5">mysql</database>
+        <lib min-version="2.7.8">libxml</lib>
+        <lib>curl</lib>
+        <lib>SimpleXML</lib>
+        <lib>iconv</lib>
+    </dependencies>
+
+    <!-- navigation -->
+    <navigation>
+        <route>news.page.index</route>
+        <order>10</order>
+        <icon>app.svg</icon>
+        <name>News</name>
+    </navigation>
+
+    <!-- admin -->
+    <admin>admin/admin</admin>
+
+    <!-- cron jobs -->
+    <cron>
+        <job>OCA\News\Cron\Updater</job>
+    </cron>
+
+    <!-- hooks -->
+    <hooks>
+        <hook>
+            <channel>OC_User::pre_deleteUser</channel>
+            <subscriber>OCA\News\Hooks\User::deleteUser</subscriber>
+        </hook>
+    </hooks>
+    <ocsid>168040</ocsid>
+</info>
diff --git a/nextcloudappstore/core/api/v1/tests/test_parser.py b/nextcloudappstore/core/api/v1/tests/test_parser.py
--- a/nextcloudappstore/core/api/v1/tests/test_parser.py
+++ b/nextcloudappstore/core/api/v1/tests/test_parser.py
@@ -6,6 +6,7 @@
     UnsupportedAppArchiveException, InvalidAppMetadataXmlException
 from nextcloudappstore.core.facades import resolve_file_relative_path, \
     read_file_contents
+from rest_framework.exceptions import APIException
 
 
 class ParserTest(TestCase):
@@ -16,6 +17,7 @@ def setUp(self):
     def test_parse_minimal(self):
         xml = self._get_test_xml('data/infoxmls/minimal.xml')
         result = parse_app_metadata(xml, self.config.info_schema,
+                                    self.config.pre_info_xslt,
                                     self.config.info_xslt)
         expected = {'app': {
             'id': 'news',
@@ -47,6 +49,20 @@ def test_validate_schema(self):
         xml = self._get_test_xml('data/infoxmls/invalid.xml')
         with (self.assertRaises(InvalidAppMetadataXmlException)):
             parse_app_metadata(xml, self.config.info_schema,
+                               self.config.pre_info_xslt,
+                               self.config.info_xslt)
+
+    def test_fixes_xml(self):
+        xml = self._get_test_xml('data/infoxmls/news.xml')
+        parse_app_metadata(xml, self.config.info_schema,
+                           self.config.pre_info_xslt,
+                           self.config.info_xslt)
+
+    def test_broken_xml(self):
+        xml = self._get_test_xml('data/infoxmls/broken-xml.xml')
+        with (self.assertRaises(APIException)):
+            parse_app_metadata(xml, self.config.info_schema,
+                               self.config.pre_info_xslt,
                                self.config.info_xslt)
 
     def test_extract_gunzip_info(self):
@@ -89,6 +105,7 @@ def test_extract_zip(self):
     def test_map_data(self):
         full = self._get_test_xml('data/infoxmls/full.xml')
         result = parse_app_metadata(full, self.config.info_schema,
+                                    self.config.pre_info_xslt,
                                     self.config.info_xslt)
         expected = {'app': {
             'id': 'news',
diff --git a/nextcloudappstore/core/api/v1/tests/test_release_importer.py b/nextcloudappstore/core/api/v1/tests/test_release_importer.py
--- a/nextcloudappstore/core/api/v1/tests/test_release_importer.py
+++ b/nextcloudappstore/core/api/v1/tests/test_release_importer.py
@@ -29,6 +29,7 @@ def test_import_minimal(self):
         self.app.save()
 
         result = parse_app_metadata(self.min, self.config.info_schema,
+                                    self.config.pre_info_xslt,
                                     self.config.info_xslt)
         self.importer.import_release(result)
         app = App.objects.get(pk='news')
@@ -67,6 +68,7 @@ def test_full(self):
         Database.objects.create(id='pgsql')
         Database.objects.create(id='mysql')
         result = parse_app_metadata(self.full, self.config.info_schema,
+                                    self.config.pre_info_xslt,
                                     self.config.info_xslt)
         self.importer.import_release(result)
         app = App.objects.get(pk='news')
@@ -117,6 +119,7 @@ def test_full(self):
 
     def test_release_update(self):
         result = parse_app_metadata(self.min, self.config.info_schema,
+                                    self.config.pre_info_xslt,
                                     self.config.info_xslt)
         self.importer.import_release(result)
         result['app']['website'] = 'https://website.com'
@@ -126,6 +129,7 @@ def test_release_update(self):
 
     def test_release_no_update(self):
         result = parse_app_metadata(self.min, self.config.info_schema,
+                                    self.config.pre_info_xslt,
                                     self.config.info_xslt)
         self.importer.import_release(result)
         result['app']['website'] = 'https://website.com'