Repository: aboutcode-org/purldb (language: python)

Issue:
Index conan packages
Conan is a good source of C/C++ packages. See https://github.com/conan-io/conan-center-index/


Discussion / hints:

Fix patch:
diff --git a/minecode/visitors/conan.py b/minecode/visitors/conan.py
new file mode 100644
index 00000000..15ec2678
--- /dev/null
+++ b/minecode/visitors/conan.py
@@ -0,0 +1,159 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+import logging
+
+import requests
+import saneyaml
+from packagedcode.conan import ConanFileHandler
+from packageurl import PackageURL
+
+from minecode import priority_router
+from packagedb.models import PackageContentType
+
+"""
+Collect Conan packages from Conan Central.
+"""
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def get_yaml_response(url):
+    """
+    Fetch YAML content from the url and return it as a dictionary.
+    """
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        content = response.content.decode("utf-8")
+        return saneyaml.load(content)
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")
+
+
+def get_conan_recipe(name, version):
+    """
+    Return the contents of the `conanfile.py` and `conandata.yml` file for
+    the conan package described by name and version string.
+    """
+    base_index_url = (
+        "https://raw.githubusercontent.com/conan-io/"
+        "conan-center-index/master/recipes/"
+    )
+
+    conan_central_config_url = f"{base_index_url}/{name}/config.yml"
+    config = get_yaml_response(conan_central_config_url)
+    if not config:
+        return None, None
+
+    versions = config.get("versions", {})
+    recipe_location = versions.get(version, {})
+    folder = recipe_location.get("folder")
+
+    folder = recipe_location.get("folder")
+    if not folder:
+        logger.error(f"No folder found for version {version} of package {name}")
+        return None, None
+
+    conanfile_py_url = f"{base_index_url}/{name}/{folder}/conanfile.py"
+    conandata_yml_url = f"{base_index_url}/{name}/{folder}/conandata.yml"
+
+    conandata = get_yaml_response(conandata_yml_url)
+
+    try:
+        response = requests.get(conanfile_py_url)
+        response.raise_for_status()
+        conanfile = response.text
+    except requests.exceptions.HTTPError as err:
+        logger.error(
+            f"HTTP error occurred while fetching conanfile.py for {name} {version}: {err}"
+        )
+        conanfile = None
+
+    return conanfile, conandata
+
+
+def get_download_info(conandata, version):
+    """
+    Return download_url and SHA256 hash from `conandata.yml`.
+    """
+    sources = conandata.get("sources", {})
+    pkg_data = sources.get(version, {})
+
+    download_url = pkg_data.get("url")
+    sha256 = pkg_data.get("sha256")
+
+    if isinstance(download_url, list):
+        download_url = download_url[0]
+
+    return download_url, sha256
+
+
+def map_conan_package(package_url):
+    """
+    Add a conan `package_url` to the PackageDB.
+
+    Return an error string if any errors are encountered during the process
+    """
+    from minecode.model_utils import add_package_to_scan_queue
+    from minecode.model_utils import merge_or_create_package
+
+    conanfile, conandata = get_conan_recipe(
+        name=package_url.name,
+        version=package_url.version,
+    )
+
+    download_url, sha256 = get_download_info(conandata, package_url.version)
+
+    if not conanfile:
+        error = f"Package does not exist on conan central: {package_url}"
+        logger.error(error)
+        return error
+    if not download_url:
+        error = f"Package download_url does not exist on conan central: {package_url}"
+        logger.error(error)
+        return error
+
+    package = ConanFileHandler._parse(conan_recipe=conanfile)
+    package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE
+    package.version = package_url.version
+    package.download_url = download_url
+    package.sha256 = sha256
+
+    db_package, _, _, error = merge_or_create_package(package, visit_level=0)
+
+    # Submit package for scanning
+    if db_package:
+        add_package_to_scan_queue(db_package)
+
+    return error
+
+
+@priority_router.route("pkg:conan/.*")
+def process_request(purl_str):
+    """
+    Process `priority_resource_uri` containing a conan Package URL (PURL) as a
+    URI.
+
+    This involves obtaining Package information for the PURL from
+    https://github.com/conan-io/conan-center-index and using it to create a new
+    PackageDB entry. The package is then added to the scan queue afterwards.
+    """
+    package_url = PackageURL.from_string(purl_str)
+    if not package_url.version:
+        return
+
+    error_msg = map_conan_package(package_url)
+
+    if error_msg:
+        return error_msg


Test patch:
diff --git a/minecode/tests/test_conan.py b/minecode/tests/test_conan.py
new file mode 100644
index 00000000..f15b9611
--- /dev/null
+++ b/minecode/tests/test_conan.py
@@ -0,0 +1,108 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+
+import os
+
+import saneyaml
+from django.test import TestCase
+from mock import patch
+from packageurl import PackageURL
+
+import packagedb
+from minecode.utils_test import JsonBasedTesting
+from minecode.visitors import conan
+
+
+class ConanPriorityQueueTests(JsonBasedTesting, TestCase):
+    test_data_dir = os.path.join(os.path.dirname(__file__), "testfiles")
+
+    def setUp(self):
+        super(ConanPriorityQueueTests, self).setUp()
+        self.package_url1 = PackageURL.from_string("pkg:conan/zlib@1.3.1")
+        zlib_conanfile_loc = self.get_test_loc("conan/zlib/manifest/conanfile.py")
+        zlib_conandata_loc = self.get_test_loc("conan/zlib/manifest/conandata.yml")
+        zlib_config_loc = self.get_test_loc("conan/zlib/manifest/config.yml")
+
+        with open(zlib_conanfile_loc) as f:
+            self.zlib_conanfile_contents = f.read()
+
+        with open(zlib_config_loc) as f:
+            self.zlib_config_contents = f.read()
+
+        with open(zlib_conandata_loc) as f:
+            self.zlib_conandata_contents = f.read()
+
+        self.zlib_conandata_contents_dict = saneyaml.load(self.zlib_conandata_contents)
+
+    @patch("requests.get")
+    def test_get_conan_recipe(self, mock_get):
+        mock_get.side_effect = [
+            type(
+                "Response",
+                (),
+                {
+                    "content": self.zlib_config_contents.encode(),
+                    "raise_for_status": lambda: None,
+                },
+            ),
+            type(
+                "Response",
+                (),
+                {
+                    "content": self.zlib_conandata_contents.encode(),
+                    "raise_for_status": lambda: None,
+                },
+            ),
+            type(
+                "Response",
+                (),
+                {
+                    "text": self.zlib_conanfile_contents,
+                    "raise_for_status": lambda: None,
+                },
+            ),
+        ]
+        result_conanfile, result_conandata = conan.get_conan_recipe(
+            self.package_url1.name, self.package_url1.version
+        )
+
+        self.assertEqual(result_conanfile, self.zlib_conanfile_contents)
+        self.assertEqual(result_conandata, self.zlib_conandata_contents_dict)
+
+    def test_get_download_info(self):
+        result_download_url, result_sha256 = conan.get_download_info(
+            self.zlib_conandata_contents_dict, self.package_url1.version
+        )
+        expected_zlib_download_url = "https://zlib.net/fossils/zlib-1.3.1.tar.gz"
+        expected_zlib_sha256 = (
+            "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23"
+        )
+
+        self.assertEqual(result_download_url, expected_zlib_download_url)
+        self.assertEqual(result_sha256, expected_zlib_sha256)
+
+    @patch("minecode.visitors.conan.get_conan_recipe")
+    def test_map_conan_package(self, mock_get_conan_recipe):
+        mock_get_conan_recipe.return_value = (
+            self.zlib_conanfile_contents,
+            self.zlib_conandata_contents_dict,
+        )
+
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(package_count, 0)
+
+        conan.map_conan_package(self.package_url1)
+        package_count = packagedb.models.Package.objects.all().count()
+        self.assertEqual(package_count, 1)
+        package = packagedb.models.Package.objects.all().first()
+        expected_zlib_download_url = "https://zlib.net/fossils/zlib-1.3.1.tar.gz"
+
+        self.assertEqual(package.purl, str(self.package_url1))
+        self.assertEqual(package.download_url, expected_zlib_download_url)
diff --git a/minecode/tests/testfiles/conan/zlib/manifest.ABOUT b/minecode/tests/testfiles/conan/zlib/manifest.ABOUT
new file mode 100644
index 00000000..f5615c81
--- /dev/null
+++ b/minecode/tests/testfiles/conan/zlib/manifest.ABOUT
@@ -0,0 +1,9 @@
+about_resource: manifest/
+name: conan-center-index
+version: 30c5f6b4aeeb9bce3614b9b2f1db7ea3324d8cdc
+download_url: https://github.com/conan-io/conan-center-index/tree/30c5f6b4aeeb9bce3614b9b2f1db7ea3324d8cdc/recipes/zlib/
+vcs_repository: https://github.com/conan-io/conan-center-index.git
+license: mit
+license_text: manifest.LICENSE
+description: Recipes for the ConanCenter repository.
+notes: recipe used for testing conan package indexing 
\ No newline at end of file
diff --git a/minecode/tests/testfiles/conan/zlib/manifest.LICENSE b/minecode/tests/testfiles/conan/zlib/manifest.LICENSE
new file mode 100644
index 00000000..2b6d3822
--- /dev/null
+++ b/minecode/tests/testfiles/conan/zlib/manifest.LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Conan.io
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/minecode/tests/testfiles/conan/zlib/manifest/conandata.yml b/minecode/tests/testfiles/conan/zlib/manifest/conandata.yml
new file mode 100644
index 00000000..5010531f
--- /dev/null
+++ b/minecode/tests/testfiles/conan/zlib/manifest/conandata.yml
@@ -0,0 +1,57 @@
+sources:
+  "1.3.1":
+    url:
+      - "https://zlib.net/fossils/zlib-1.3.1.tar.gz"
+      - "https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.1.tar.gz"
+    sha256: "9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23"
+  "1.3":
+    url:
+      - "https://zlib.net/fossils/zlib-1.3.tar.gz"
+      - "https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz"
+    sha256: "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e"
+  "1.2.13":
+    url:
+      - "https://zlib.net/fossils/zlib-1.2.13.tar.gz"
+      - "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.gz"
+    sha256: "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30"
+  "1.2.12":
+    url: "https://zlib.net/fossils/zlib-1.2.12.tar.gz"
+    sha256: "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9"
+  "1.2.11":
+    url: "https://zlib.net/fossils/zlib-1.2.11.tar.gz"
+    sha256: "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1"
+patches:
+  "1.3.1":
+    - patch_file: "patches/1.3.1/0001-fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix"
+      patch_type: "conan"
+  "1.3":
+    - patch_file: "patches/1.3/0001-fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix, disable building examples"
+      patch_type: "conan"
+  "1.2.13":
+    - patch_file: "patches/1.2.13/0001-Fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix, disable building examples"
+      patch_type: "conan"
+  "1.2.12":
+    - patch_file: "patches/1.2.x/0001-fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix, disable building examples"
+      patch_type: "conan"
+    - patch_file: "patches/1.2.x/0004-Fix-a-bug-when-getting-a-gzip-header-extra-field-wit.patch"
+      patch_description: "CVE-2022-37434: Fix a bug when getting a gzip header extra field with inflate()"
+      patch_type: "vulnerability"
+      patch_source: "https://github.com/madler/zlib/commit/eff308af425b67093bab25f80f1ae950166bece1"
+      sha256: "15e3c177dc2a034a22e02490a97ba5b1719aae3f8129a06c16d727b661d1650f"
+    - patch_file: "patches/1.2.x/0005-Fix-extra-field-processing-bug-that-dereferences-NUL.patch"
+      patch_description: "CVE-2022-37434: Fix extra field processing bug that dereferences NULL state->head"
+      patch_type: "vulnerability"
+      patch_source: "https://github.com/madler/zlib/commit/1eb7682f845ac9e9bf9ae35bbfb3bad5dacbd91d"
+      sha256: "cdd69eb3251728b1875c8ecae6427b50aa750b4045ef984ab79b6c07b7e6dd3a"
+  "1.2.11":
+    - patch_file: "patches/1.2.x/0001-fix-cmake.patch"
+      patch_description: "separate static/shared builds, disable debug suffix, disable building examples"
+      patch_type: "conan"
+    - patch_file: "patches/1.2.x/0003-gzguts-fix-widechar-condition.patch"
+      patch_description: "fix condition for WIDECHAR usage"
+      patch_type: "portability"
+      patch_source: "https://github.com/madler/zlib/issues/268"
\ No newline at end of file
diff --git a/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py b/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py
new file mode 100644
index 00000000..72f2e5fc
--- /dev/null
+++ b/minecode/tests/testfiles/conan/zlib/manifest/conanfile.py
@@ -0,0 +1,107 @@
+class ConanFile:
+    pass
+
+required_conan_version = ">=1.53.0"
+
+
+class ZlibConan(ConanFile):
+    name = "zlib"
+    package_type = "library"
+    url = "https://github.com/conan-io/conan-center-index"
+    homepage = "https://zlib.net"
+    license = "Zlib"
+    description = ("A Massively Spiffy Yet Delicately Unobtrusive Compression Library "
+                   "(Also Free, Not to Mention Unencumbered by Patents)")
+    topics = ("zlib", "compression")
+
+    settings = "os", "arch", "compiler", "build_type"
+    options = {
+        "shared": [True, False],
+        "fPIC": [True, False],
+    }
+    default_options = {
+        "shared": False,
+        "fPIC": True,
+    }
+
+    @property
+    def _is_mingw(self):
+        return self.settings.os == "Windows" and self.settings.compiler == "gcc"
+
+    def export_sources(self):
+        export_conandata_patches(self)
+
+    def config_options(self):
+        if self.settings.os == "Windows":
+            del self.options.fPIC
+
+    def configure(self):
+        if self.options.shared:
+            self.options.rm_safe("fPIC")
+        self.settings.rm_safe("compiler.libcxx")
+        self.settings.rm_safe("compiler.cppstd")
+
+    def layout(self):
+        cmake_layout(self, src_folder="src")
+
+    def source(self):
+        get(self, **self.conan_data["sources"][self.version],
+            destination=self.source_folder, strip_root=True)
+
+    def generate(self):
+        tc = CMakeToolchain(self)
+        tc.variables["SKIP_INSTALL_ALL"] = False
+        tc.variables["SKIP_INSTALL_LIBRARIES"] = False
+        tc.variables["SKIP_INSTALL_HEADERS"] = False
+        tc.variables["SKIP_INSTALL_FILES"] = True
+        # Correct for misuse of "${CMAKE_INSTALL_PREFIX}/" in CMakeLists.txt
+        tc.variables["INSTALL_LIB_DIR"] = "lib"
+        tc.variables["INSTALL_INC_DIR"] = "include"
+        tc.variables["ZLIB_BUILD_EXAMPLES"] = False
+        tc.generate()
+
+    def _patch_sources(self):
+        apply_conandata_patches(self)
+
+        is_apple_clang12 = self.settings.compiler == "apple-clang" and Version(self.settings.compiler.version) >= "12.0"
+        if not is_apple_clang12:
+            for filename in ['zconf.h', 'zconf.h.cmakein', 'zconf.h.in']:
+                filepath = os.path.join(self.source_folder, filename)
+                replace_in_file(self, filepath,
+                                      '#ifdef HAVE_UNISTD_H    '
+                                      '/* may be set to #if 1 by ./configure */',
+                                      '#if defined(HAVE_UNISTD_H) && (1-HAVE_UNISTD_H-1 != 0)')
+                replace_in_file(self, filepath,
+                                      '#ifdef HAVE_STDARG_H    '
+                                      '/* may be set to #if 1 by ./configure */',
+                                      '#if defined(HAVE_STDARG_H) && (1-HAVE_STDARG_H-1 != 0)')
+
+    def build(self):
+        self._patch_sources()
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def _extract_license(self):
+        tmp = load(self, os.path.join(self.source_folder, "zlib.h"))
+        license_contents = tmp[2:tmp.find("*/", 1)]
+        return license_contents
+
+    def package(self):
+        save(self, os.path.join(self.package_folder, "licenses", "LICENSE"), self._extract_license())
+        cmake = CMake(self)
+        cmake.install()
+
+    def package_info(self):
+        self.cpp_info.set_property("cmake_find_mode", "both")
+        self.cpp_info.set_property("cmake_file_name", "ZLIB")
+        self.cpp_info.set_property("cmake_target_name", "ZLIB::ZLIB")
+        self.cpp_info.set_property("pkg_config_name", "zlib")
+        if self.settings.os == "Windows" and not self._is_mingw:
+            libname = "zdll" if self.options.shared else "zlib"
+        else:
+            libname = "z"
+        self.cpp_info.libs = [libname]
+
+        self.cpp_info.names["cmake_find_package"] = "ZLIB"
+        self.cpp_info.names["cmake_find_package_multi"] = "ZLIB"
\ No newline at end of file
diff --git a/minecode/tests/testfiles/conan/zlib/manifest/config.yml b/minecode/tests/testfiles/conan/zlib/manifest/config.yml
new file mode 100644
index 00000000..e42c3dfb
--- /dev/null
+++ b/minecode/tests/testfiles/conan/zlib/manifest/config.yml
@@ -0,0 +1,11 @@
+versions:
+  "1.3.1":
+    folder: all
+  "1.3":
+    folder: all
+  "1.2.13":
+    folder: all
+  "1.2.12":
+    folder: all
+  "1.2.11":
+    folder: all
\ No newline at end of file