Repository: SFDO-Tooling/CumulusCI
Pull request #3727 (2024-01-03 11:39:23)

Identifies and retrieves the non source trackable Metadata type components 
[W-14699376](https://gus.lightning.force.com/lightning/r/ADM_Work__c/a07EE00001gnCZHYA2/view)
[W-14698998](https://gus.lightning.force.com/lightning/r/ADM_Work__c/a07EE00001gmf8ZYAQ/view)
[W-14698982](https://gus.lightning.force.com/lightning/r/ADM_Work__c/a07EE00001gmmuuYAA/view)


Identifies the list of non source trackable metadata types from the metadata coverage report URL. Later checks whether the metadata types are supported by the org or not. 

List the components for the individual component and then retrieves the selected components.

lakshmi2506: Hi @jkasturi-sf @kdvenkata 

Please review the PR
Thank you

Patch:
diff --git a/cumulusci/cumulusci.yml b/cumulusci/cumulusci.yml
index 9b13e8174..6dd783257 100644
--- a/cumulusci/cumulusci.yml
+++ b/cumulusci/cumulusci.yml
@@ -415,6 +415,14 @@ tasks:
         description: Prints the metadata types in a project
         class_path: cumulusci.tasks.util.ListMetadataTypes
         group: Salesforce Metadata
+    list_nonsource_trackable_components:
+        description: List the components of non source trackable Metadata types.
+        class_path: cumulusci.tasks.salesforce.nonsourcetracking.ListComponents
+        group: Salesforce Metadata
+    list_nonsource_trackable_metadatatypes:
+        description: Returns non source trackable metadata types supported by org
+        class_path: cumulusci.tasks.salesforce.nonsourcetracking.ListNonSourceTrackable
+        group: Salesforce Metadata
     meta_xml_apiversion:
         description: Set the API version in ``*meta.xml`` files
         class_path: cumulusci.tasks.metaxml.UpdateApi
@@ -504,6 +512,10 @@ tasks:
         description: Retrieve changed components from a scratch org
         class_path: cumulusci.tasks.salesforce.sourcetracking.RetrieveChanges
         group: Salesforce Metadata
+    retrieve_nonsource_trackable:
+        description: Retrieves the non source trackable components filtered
+        class_path: cumulusci.tasks.salesforce.nonsourcetracking.RetrieveComponents
+        group: Salesforce Metadata
     retrieve_qa_config:
         description: Retrieves the current changes in the scratch org into unpackaged/config/qa
         class_path: cumulusci.tasks.salesforce.sourcetracking.RetrieveChanges
diff --git a/cumulusci/tasks/salesforce/DescribeMetadataTypes.py b/cumulusci/tasks/salesforce/DescribeMetadataTypes.py
index 0b2d4f4e1..8c0e237e8 100644
--- a/cumulusci/tasks/salesforce/DescribeMetadataTypes.py
+++ b/cumulusci/tasks/salesforce/DescribeMetadataTypes.py
@@ -23,4 +23,4 @@ class DescribeMetadataTypes(BaseRetrieveMetadata):
     def _run_task(self):
         api_object = self._get_api()
         self.return_values = api_object()
-        self.logger.info("Metadata Types supported by org:\n" + str(self.return_values))
+        return self.return_values
diff --git a/cumulusci/tasks/salesforce/nonsourcetracking.py b/cumulusci/tasks/salesforce/nonsourcetracking.py
new file mode 100644
index 000000000..d8c5881e1
--- /dev/null
+++ b/cumulusci/tasks/salesforce/nonsourcetracking.py
@@ -0,0 +1,248 @@
+import json
+import os
+
+import requests
+import sarge
+
+from cumulusci.core.config import TaskConfig
+from cumulusci.core.exceptions import CumulusCIException, SfdxOrgException
+from cumulusci.core.sfdx import sfdx
+from cumulusci.core.utils import process_list_arg
+from cumulusci.tasks.salesforce import (
+    BaseRetrieveMetadata,
+    BaseSalesforceApiTask,
+    DescribeMetadataTypes,
+)
+from cumulusci.tasks.salesforce.sourcetracking import ListChanges, retrieve_components
+
+nl = "\n"
+
+
+class ListNonSourceTrackable(BaseSalesforceApiTask):
+
+    task_options = {
+        "api_version": {
+            "description": "Override the API version used to list metadatatypes",
+        },
+    }
+
+    def _init_task(self):
+        super()._init_task()
+        if "api_version" not in self.options:
+            self.options[
+                "api_version"
+            ] = self.project_config.project__package__api_version
+
+    def get_types_details(self, api_version):
+        # The Metadata coverage report: https://developer.salesforce.com/docs/metadata-coverage/{version} is created from
+        # the below URL. (So the api versions are allowed based on those report ranges)
+        url = f"https://dx-extended-coverage.my.salesforce-sites.com/services/apexrest/report?version={api_version}"
+        response = requests.get(url)
+        if response.status_code == 200:
+            json_response = response.json()
+            return json_response
+        else:
+            raise CumulusCIException(
+                f"Failed to retrieve response with status code {response.status_code}"
+            )
+
+    def _run_task(self):
+        metadatatypes_details = self.get_types_details(self.options["api_version"])[
+            "types"
+        ]
+        all_nonsource_types = []
+        for md_type, details in metadatatypes_details.items():
+            if not details["channels"]:
+                raise CumulusCIException(
+                    f"Api version {self.options['api_version']} not supported"
+                )
+            if (
+                details["channels"]["sourceTracking"] is False
+                and details["channels"]["metadataApi"] is True
+            ):
+                all_nonsource_types.append(md_type)
+
+        types_supported = DescribeMetadataTypes(
+            org_config=self.org_config,
+            project_config=self.project_config,
+            task_config=self.task_config,
+        )._run_task()
+
+        self.return_values = []
+        for md_type in all_nonsource_types:
+            if md_type in types_supported:
+                self.return_values.append(md_type)
+
+        if self.return_values:
+            self.return_values.sort()
+
+        self.logger.info(
+            f"Non source trackable Metadata types supported by org: \n{self.return_values}"
+        )
+        return self.return_values
+
+
+class ListComponents(BaseSalesforceApiTask):
+    task_options = {
+        "api_version": {
+            "description": "Override the API version used to list metadatatypes",
+        },
+        "metadata_types": {"description": "A comma-separated list of metadata types."},
+    }
+
+    def _init_task(self):
+        super()._init_task()
+
+    def _init_options(self, kwargs):
+        super(ListComponents, self)._init_options(kwargs)
+        if "api_version" not in self.options:
+            self.options[
+                "api_version"
+            ] = self.project_config.project__package__api_version
+        self.options["metadata_types"] = process_list_arg(
+            self.options.get("metadata_types", [])
+        )
+
+    def _get_components(self):
+        task_config = TaskConfig(
+            {"options": {"api_version": self.options["api_version"]}}
+        )
+        if not self.options["metadata_types"]:
+            metadata_types = ListNonSourceTrackable(
+                org_config=self.org_config,
+                project_config=self.project_config,
+                task_config=task_config,
+            )._run_task()
+            self.options["metadata_types"] = metadata_types
+        list_components = []
+        for md_type in self.options["metadata_types"]:
+            p: sarge.Command = sfdx(
+                "force:mdapi:listmetadata",
+                access_token=self.org_config.access_token,
+                log_note="Listing components",
+                args=[
+                    "-a",
+                    str(self.options["api_version"]),
+                    "-m",
+                    str(md_type),
+                    "--json",
+                ],
+                env={"SFDX_INSTANCE_URL": self.org_config.instance_url},
+            )
+            stdout, stderr = p.stdout_text.read(), p.stderr_text.read()
+
+            if p.returncode:
+                message = f"\nstderr:\n{nl.join(stderr)}"
+                message += f"\nstdout:\n{nl.join(stdout)}"
+                raise SfdxOrgException(message)
+            else:
+                result = json.loads(stdout)["result"]
+                if result:
+                    for cmp in result:
+                        change_dict = {
+                            "MemberType": md_type,
+                            "MemberName": cmp["fullName"],
+                            "lastModifiedByName": cmp["lastModifiedByName"],
+                            "lastModifiedDate": cmp["lastModifiedDate"],
+                        }
+                        if change_dict not in list_components:
+                            list_components.append(change_dict)
+
+        return list_components
+
+    def _run_task(self):
+        self.return_values = self._get_components()
+        self.logger.info(
+            f"Found {len(self.return_values)} non source trackable components in the org for the given types."
+        )
+        for change in self.return_values:
+            self.logger.info("{MemberType}: {MemberName}".format(**change))
+        return self.return_values
+
+
+retrieve_components_task_options = ListComponents.task_options.copy()
+retrieve_components_task_options["path"] = {
+    "description": "The path to write the retrieved metadata",
+    "required": False,
+}
+retrieve_components_task_options["include"] = {
+    "description": "Components will be included if one of these names"
+    "is part of either the metadata type or name. "
+    "Example: ``-o include CustomField,Admin`` matches both "
+    "``CustomField: Favorite_Color__c`` and ``Profile: Admin``"
+}
+retrieve_components_task_options["exclude"] = {
+    "description": "Exclude components matching this name."
+}
+retrieve_components_task_options[
+    "namespace_tokenize"
+] = BaseRetrieveMetadata.task_options["namespace_tokenize"]
+
+
+class RetrieveComponents(ListComponents, BaseSalesforceApiTask):
+    task_options = retrieve_components_task_options
+
+    def _init_options(self, kwargs):
+        super(RetrieveComponents, self)._init_options(kwargs)
+        self.options["include"] = process_list_arg(self.options.get("include", []))
+        self.options["exclude"] = process_list_arg(self.options.get("exclude", []))
+        self._include = self.options["include"]
+        self._exclude = self.options["exclude"]
+        self._exclude.extend(self.project_config.project__source__ignore or [])
+
+        package_directories = []
+        default_package_directory = None
+        if os.path.exists("sfdx-project.json"):
+            with open("sfdx-project.json", "r", encoding="utf-8") as f:
+                sfdx_project = json.load(f)
+                for package_directory in sfdx_project.get("packageDirectories", []):
+                    package_directories.append(package_directory["path"])
+                    if package_directory.get("default"):
+                        default_package_directory = package_directory["path"]
+
+        path = self.options.get("path")
+        if path is None:
+            # set default path to src for mdapi format,
+            # or the default package directory from sfdx-project.json for dx format
+            if (
+                default_package_directory
+                and self.project_config.project__source_format == "sfdx"
+            ):
+                path = default_package_directory
+                md_format = False
+            else:
+                path = "src"
+                md_format = True
+        else:
+            md_format = path not in package_directories
+        self.md_format = md_format
+        self.options["path"] = path
+
+    def _run_task(self):
+        components = self._get_components()
+        filtered, ignored = ListChanges._filter_changes(self, components)
+        if not filtered:
+            self.logger.info("No components to retrieve")
+            return
+        for cmp in filtered:
+            self.logger.info("{MemberType}: {MemberName}".format(**cmp))
+
+        target = os.path.realpath(self.options["path"])
+        package_xml_opts = {}
+        if self.options["path"] == "src":
+            package_xml_opts.update(
+                {
+                    "package_name": self.project_config.project__package__name,
+                    "install_class": self.project_config.project__package__install_class,
+                    "uninstall_class": self.project_config.project__package__uninstall_class,
+                }
+            )
+        retrieve_components(
+            filtered,
+            self.org_config,
+            target,
+            md_format=self.md_format,
+            namespace_tokenize=self.options.get("namespace_tokenize"),
+            api_version=self.options["api_version"],
+            extra_package_xml_opts=package_xml_opts,
+        )


Test patch:
diff --git a/cumulusci/tasks/salesforce/tests/test_nonsourcetracking.py b/cumulusci/tasks/salesforce/tests/test_nonsourcetracking.py
new file mode 100644
index 000000000..a9dde1ca5
--- /dev/null
+++ b/cumulusci/tasks/salesforce/tests/test_nonsourcetracking.py
@@ -0,0 +1,271 @@
+import io
+import json
+import os
+from unittest import mock
+
+import pytest
+import responses
+
+from cumulusci.core.exceptions import CumulusCIException, SfdxOrgException
+from cumulusci.tasks.salesforce import DescribeMetadataTypes
+from cumulusci.tasks.salesforce.nonsourcetracking import (
+    ListComponents,
+    ListNonSourceTrackable,
+    RetrieveComponents,
+)
+from cumulusci.tests.util import create_project_config
+from cumulusci.utils import temporary_dir
+
+
+class TestListNonSourceTrackable:
+    @pytest.mark.parametrize(
+        "json_data, expected_status,error_msg",
+        [
+            (None, 404, "Failed to retrieve response with status code 404"),
+            (
+                {"types": {"Scontrol": {"channels": {}}}},
+                200,
+                "Api version 44.0 not supported",
+            ),
+        ],
+    )
+    @responses.activate
+    def test_run_get_types_details(
+        self, create_task_fixture, json_data, expected_status, error_msg
+    ):
+        options = {"api_version": 44.0}
+        responses.add(
+            "GET",
+            f"https://dx-extended-coverage.my.salesforce-sites.com/services/apexrest/report?version={options['api_version']}",
+            json=json_data,
+            status=expected_status,
+        )
+        task = create_task_fixture(ListNonSourceTrackable, options)
+        task._init_task()
+        with pytest.raises(CumulusCIException, match=error_msg):
+            task._run_task()
+
+    @responses.activate
+    def test_run_task_list_metadatatypes(self, create_task_fixture):
+        options = {"api_version": 44.0}
+        return_value = {
+            "types": {
+                "Scontrol": {
+                    "channels": {"sourceTracking": False, "metadataApi": True}
+                },
+                "SharingRules": {
+                    "channels": {"sourceTracking": False, "metadataApi": True}
+                },
+                "ApexClass": {
+                    "channels": {"sourceTracking": True, "metadataApi": True}
+                },
+                "CustomObject": {
+                    "channels": {"sourceTracking": True, "metadataApi": True}
+                },
+                "MessagingChannel": {
+                    "channels": {"sourceTracking": True, "metadataApi": True}
+                },
+            }
+        }
+        responses.add(
+            "GET",
+            f"https://dx-extended-coverage.my.salesforce-sites.com/services/apexrest/report?version={options['api_version']}",
+            json=return_value,
+            status=200,
+        )
+
+        task = create_task_fixture(ListNonSourceTrackable, options)
+        task._init_task()
+        with mock.patch.object(
+            DescribeMetadataTypes,
+            "_run_task",
+            return_value=["Scontrol", "ApexClass", "CustomObject", "SharingRules"],
+        ):
+            non_source_trackable = task._run_task()
+            assert non_source_trackable == ["Scontrol", "SharingRules"]
+
+
+@mock.patch("sarge.Command")
+class TestListComponents:
+    @pytest.mark.parametrize(
+        "return_code, result",
+        [
+            (
+                0,
+                b"""{
+  "status": 0,
+  "result": [],
+  "warnings": [
+    "No metadata found for type: SharingRules"
+  ]
+}""",
+            ),
+            (1, b""),
+        ],
+    )
+    def test_check_sfdx_output(self, cmd, create_task_fixture, return_code, result):
+        options = {"api_version": 44.0}
+        task = create_task_fixture(ListComponents, options)
+        cmd.return_value = mock.Mock(
+            stderr=io.BytesIO(b""), stdout=io.BytesIO(result), returncode=return_code
+        )
+        task._init_task()
+        with mock.patch.object(
+            ListNonSourceTrackable, "_run_task", return_value=["SharingRules"]
+        ):
+            if return_code:
+                with pytest.raises(SfdxOrgException):
+                    task._run_task()
+            else:
+                assert task._run_task() == []
+
+    @pytest.mark.parametrize(
+        "options",
+        [
+            {"api_version": 44.0, "metadata_types": "FlowDefinition"},
+            {
+                "api_version": 44.0,
+            },
+        ],
+    )
+    def test_check_sfdx_result(self, cmd, create_task_fixture, options):
+        task = create_task_fixture(ListComponents, options)
+        result = b"""{
+  "status": 0,
+  "result": [
+    {
+      "createdById": "0051y00000OdeZBAAZ",
+      "createdByName": "User User",
+      "createdDate": "2024-01-02T06:50:03.000Z",
+      "fileName": "flowDefinitions/alpha.flowDefinition",
+      "fullName": "alpha",
+      "id": "3001y000000ERX6AAO",
+      "lastModifiedById": "0051y00000OdeZBAAZ",
+      "lastModifiedByName": "User User",
+      "lastModifiedDate": "2024-01-02T06:50:07.000Z",
+      "manageableState": "unmanaged",
+      "namespacePrefix": "myalpha",
+      "type": "FlowDefinition"
+    },{
+      "createdById": "0051y00000OdeZBAAZ",
+      "createdByName": "User User",
+      "createdDate": "2024-01-02T06:50:03.000Z",
+      "fileName": "flowDefinitions/beta.flowDefinition",
+      "fullName": "beta",
+      "id": "3001y000000ERX6AAO",
+      "lastModifiedById": "0051y00000OdeZBAAZ",
+      "lastModifiedByName": "User User",
+      "lastModifiedDate": "2024-01-02T06:50:07.000Z",
+      "manageableState": "unmanaged",
+      "namespacePrefix": "myalpha",
+      "type": "FlowDefinition"
+    }
+  ],
+  "warnings": []
+}"""
+        cmd.return_value = mock.Mock(
+            stderr=io.BytesIO(b""), stdout=io.BytesIO(result), returncode=0
+        )
+        messages = []
+        task._init_task()
+        task.logger = mock.Mock()
+        task.logger.info = messages.append
+        if "metadata_types" in options:
+            components = task._run_task()
+            assert cmd.call_count == 1
+            assert (
+                "sfdx force:mdapi:listmetadata -a 44.0 -m FlowDefinition --json"
+                in cmd.call_args[0][0]
+            )
+            assert components == [
+                {
+                    "MemberType": "FlowDefinition",
+                    "MemberName": "alpha",
+                    "lastModifiedByName": "User User",
+                    "lastModifiedDate": "2024-01-02T06:50:07.000Z",
+                },
+                {
+                    "MemberType": "FlowDefinition",
+                    "MemberName": "beta",
+                    "lastModifiedByName": "User User",
+                    "lastModifiedDate": "2024-01-02T06:50:07.000Z",
+                },
+            ]
+            assert (
+                "Found 2 non source trackable components in the org for the given types."
+                in messages
+            )
+        else:
+            with mock.patch.object(
+                ListNonSourceTrackable,
+                "_run_task",
+                return_value=["Index"],
+            ):
+                task._run_task()
+                assert cmd.call_count == 1
+                assert (
+                    "sfdx force:mdapi:listmetadata -a 44.0 -m Index --json"
+                    in cmd.call_args[0][0]
+                )
+
+
+@mock.patch("cumulusci.tasks.salesforce.sourcetracking.sfdx")
+class TestRetrieveComponents:
+    def test_init_options__sfdx_format(self, sfdx, create_task_fixture):
+        with temporary_dir():
+            project_config = create_project_config()
+            project_config.project__source_format = "sfdx"
+            with open("sfdx-project.json", "w") as f:
+                json.dump(
+                    {"packageDirectories": [{"path": "force-app", "default": True}]}, f
+                )
+            task = create_task_fixture(RetrieveComponents, {}, project_config)
+            assert not task.md_format
+            assert task.options["path"] == "force-app"
+
+    def test_run_task(self, sfdx, create_task_fixture):
+        sfdx_calls = []
+        sfdx.side_effect = lambda cmd, *args, **kw: sfdx_calls.append(cmd)
+
+        with temporary_dir():
+            task = create_task_fixture(
+                RetrieveComponents, {"include": "alpha", "namespace_tokenize": "ns"}
+            )
+            task._init_task()
+            with mock.patch.object(
+                ListComponents,
+                "_get_components",
+                return_value=[
+                    {
+                        "MemberType": "FlowDefinition",
+                        "MemberName": "alpha",
+                        "lastModifiedByName": "User User",
+                        "lastModifiedDate": "2024-01-02T06:50:07.000Z",
+                    },
+                    {
+                        "MemberType": "FlowDefinition",
+                        "MemberName": "beta",
+                        "lastModifiedByName": "User User",
+                        "lastModifiedDate": "2024-01-02T06:50:07.000Z",
+                    },
+                ],
+            ):
+                task._run_task()
+
+                assert sfdx_calls == [
+                    "force:mdapi:convert",
+                    "force:source:retrieve",
+                    "force:source:convert",
+                ]
+                assert os.path.exists(os.path.join("src", "package.xml"))
+
+    def test_run_task__no_changes(self, sfdx, create_task_fixture):
+        with temporary_dir() as path:
+            task = create_task_fixture(RetrieveComponents, {"path": path})
+            task._init_task()
+            messages = []
+            with mock.patch.object(ListComponents, "_get_components", return_value=[]):
+                task.logger = mock.Mock()
+                task.logger.info = messages.append
+                task._run_task()
+                assert "No components to retrieve" in messages