Repository: SFDO-Tooling/CumulusCI
Pull request #3650 (2023-09-13 07:05:48)

Extended 'Deploy' task to support REST API deployment
Adding functionality to existing task "Deploy" where metadata can be deployed using REST API.


Patch:
diff --git a/cumulusci/salesforce_api/rest_deploy.py b/cumulusci/salesforce_api/rest_deploy.py
new file mode 100644
index 000000000..70d532569
--- /dev/null
+++ b/cumulusci/salesforce_api/rest_deploy.py
@@ -0,0 +1,148 @@
+import base64
+import io
+import json
+import os
+import time
+import uuid
+import zipfile
+from typing import List, Union
+
+import requests
+
+PARENT_DIR_NAME = "metadata"
+
+
+class RestDeploy:
+    def __init__(
+        self,
+        task,
+        package_zip: str,
+        purge_on_delete: Union[bool, str, None],
+        check_only: bool,
+        test_level: Union[str, None],
+        run_tests: List[str],
+    ):
+        # Initialize instance variables and configuration options
+        self.api_version = task.project_config.project__package__api_version
+        self.task = task
+        assert package_zip, "Package zip should not be None"
+        if purge_on_delete is None:
+            purge_on_delete = True
+        self._set_purge_on_delete(purge_on_delete)
+        self.check_only = "true" if check_only else "false"
+        self.test_level = test_level
+        self.package_zip = package_zip
+        self.run_tests = run_tests or []
+
+    def __call__(self):
+        self._boundary = str(uuid.uuid4())
+        url = f"{self.task.org_config.instance_url}/services/data/v{self.api_version}/metadata/deployRequest"
+        headers = {
+            "Authorization": f"Bearer {self.task.org_config.access_token}",
+            "Content-Type": f"multipart/form-data; boundary={self._boundary}",
+        }
+
+        # Prepare deployment options as JSON payload
+        deploy_options = {
+            "deployOptions": {
+                "allowMissingFiles": False,
+                "autoUpdatePackage": False,
+                "checkOnly": self.check_only,
+                "ignoreWarnings": False,
+                "performRetrieve": False,
+                "purgeOnDelete": self.purge_on_delete,
+                "rollbackOnError": False,
+                "runTests": self.run_tests,
+                "singlePackage": False,
+                "testLevel": self.test_level,
+            }
+        }
+        json_payload = json.dumps(deploy_options)
+
+        # Construct the multipart/form-data request body
+        body = (
+            f"--{self._boundary}\r\n"
+            f'Content-Disposition: form-data; name="json"\r\n'
+            f"Content-Type: application/json\r\n\r\n"
+            f"{json_payload}\r\n"
+            f"--{self._boundary}\r\n"
+            f'Content-Disposition: form-data; name="file"; filename="metadata.zip"\r\n'
+            f"Content-Type: application/zip\r\n\r\n"
+        ).encode("utf-8")
+        body += self._reformat_zip(self.package_zip)
+        body += f"\r\n--{self._boundary}--\r\n".encode("utf-8")
+
+        response = requests.post(url, headers=headers, data=body)
+        response_json = response.json()
+
+        if response.status_code == 201:
+            self.task.logger.info("Deployment request successful")
+            deploy_request_id = response_json["id"]
+            self._monitor_deploy_status(deploy_request_id)
+        else:
+            self.task.logger.error(
+                f"Deployment request failed with status code {response.status_code}"
+            )
+
+    # Set the purge_on_delete attribute based on org type
+    def _set_purge_on_delete(self, purge_on_delete):
+        if not purge_on_delete or purge_on_delete == "false":
+            self.purge_on_delete = "false"
+        else:
+            self.purge_on_delete = "true"
+        # Disable purge on delete entirely for non sandbox or DE orgs as it is
+        # not allowed
+        org_type = self.task.org_config.org_type
+        is_sandbox = self.task.org_config.is_sandbox
+        if org_type != "Developer Edition" and not is_sandbox:
+            self.purge_on_delete = "false"
+
+    # Monitor the deployment status and log progress
+    def _monitor_deploy_status(self, deploy_request_id):
+        url = f"{self.task.org_config.instance_url}/services/data/v{self.api_version}/metadata/deployRequest/{deploy_request_id}?includeDetails=true"
+        headers = {"Authorization": f"Bearer {self.task.org_config.access_token}"}
+
+        while True:
+            response = requests.get(url, headers=headers)
+            response_json = response.json()
+            self.task.logger.info(
+                f"Deployment {response_json['deployResult']['status']}"
+            )
+
+            if response_json["deployResult"]["status"] not in ["InProgress", "Pending"]:
+                # Handle the case when status has Failed
+                if response_json["deployResult"]["status"] == "Failed":
+                    for failure in response_json["deployResult"]["details"][
+                        "componentFailures"
+                    ]:
+                        self.task.logger.error(self._construct_error_message(failure))
+                return
+            time.sleep(5)
+
+    # Reformat the package zip file to include parent directory
+    def _reformat_zip(self, package_zip):
+        zip_bytes = base64.b64decode(package_zip)
+        zip_stream = io.BytesIO(zip_bytes)
+        new_zip_stream = io.BytesIO()
+
+        with zipfile.ZipFile(zip_stream, "r") as zip_ref:
+            with zipfile.ZipFile(new_zip_stream, "w") as new_zip_ref:
+                for item in zip_ref.infolist():
+                    # Choice of name for parent directory is irrelevant to functioning
+                    new_item_name = os.path.join(PARENT_DIR_NAME, item.filename)
+                    file_content = zip_ref.read(item.filename)
+                    new_zip_ref.writestr(new_item_name, file_content)
+
+        new_zip_bytes = new_zip_stream.getvalue()
+        return new_zip_bytes
+
+    # Construct an error message from deployment failure details
+    def _construct_error_message(self, failure):
+        error_message = f"{str.upper(failure['problemType'])} in file {failure['fileName'][len(PARENT_DIR_NAME)+len('/'):]}: {failure['problem']}"
+
+        if failure["lineNumber"] and failure["columnNumber"]:
+            error_message += (
+                f" at line {failure['lineNumber']}:{failure['columnNumber']}"
+            )
+
+        return error_message
diff --git a/cumulusci/tasks/salesforce/Deploy.py b/cumulusci/tasks/salesforce/Deploy.py
index f84ae3ae2..4f42a8a14 100644
--- a/cumulusci/tasks/salesforce/Deploy.py
+++ b/cumulusci/tasks/salesforce/Deploy.py
@@ -13,6 +13,7 @@ from cumulusci.core.source_transforms.transforms import (
 from cumulusci.core.utils import process_bool_arg, process_list_arg
 from cumulusci.salesforce_api.metadata import ApiDeploy
 from cumulusci.salesforce_api.package_zip import MetadataPackageZipBuilder
+from cumulusci.salesforce_api.rest_deploy import RestDeploy
 from cumulusci.tasks.salesforce.BaseSalesforceMetadataApiTask import (
     BaseSalesforceMetadataApiTask,
 )
@@ -55,6 +56,7 @@ class Deploy(BaseSalesforceMetadataApiTask):
         "transforms": {
             "description": "Apply source transforms before deploying. See the CumulusCI documentation for details on how to specify transforms."
         },
+        "rest_deploy": {"description": "If True, deploy metadata using REST API"},
     }
 
     namespaces = {"sf": "http://soap.sforce.com/2006/04/metadata"}
@@ -99,6 +101,9 @@ class Deploy(BaseSalesforceMetadataApiTask):
                     f"The validation error was {str(e)}"
                 )
 
+        # Set class variable to true if rest_deploy is set to True
+        self.rest_deploy = process_bool_arg(self.options.get("rest_deploy", False))
+
     def _get_api(self, path=None):
         if not path:
             path = self.options.get("path")
@@ -110,6 +115,10 @@ class Deploy(BaseSalesforceMetadataApiTask):
             self.logger.warning("Deployment package is empty; skipping deployment.")
             return
 
+        # If rest_deploy param is set, update api_class to be RestDeploy
+        if self.rest_deploy:
+            self.api_class = RestDeploy
+
         return self.api_class(
             self,
             package_zip,


Test patch:
diff --git a/cumulusci/salesforce_api/tests/test_rest_deploy.py b/cumulusci/salesforce_api/tests/test_rest_deploy.py
new file mode 100644
index 000000000..6a3794e35
--- /dev/null
+++ b/cumulusci/salesforce_api/tests/test_rest_deploy.py
@@ -0,0 +1,261 @@
+import base64
+import io
+import unittest
+import zipfile
+from unittest.mock import MagicMock, Mock, call, patch
+
+from cumulusci.salesforce_api.rest_deploy import RestDeploy
+
+
+def generate_sample_zip_data(parent=""):
+    # Create a sample ZIP with two files
+    zip_data = io.BytesIO()
+    with zipfile.ZipFile(zip_data, "w") as zip_file:
+        zip_file.writestr(
+            f"{parent}objects/mockfile1.obj", "Sample content for mockfile1"
+        )
+        zip_file.writestr(
+            f"{parent}objects/mockfile2.obj", "Sample content for mockfile2"
+        )
+    return base64.b64encode(zip_data.getvalue()).decode("utf-8")
+
+
+class TestRestDeploy(unittest.TestCase):
+    # Setup method executed before each test method
+    def setUp(self):
+        self.mock_logger = Mock()
+        self.mock_task = MagicMock()
+        self.mock_task.logger = self.mock_logger
+        self.mock_task.org_config.instance_url = "https://example.com"
+        self.mock_task.org_config.access_token = "dummy_token"
+        self.mock_task.project_config.project__package__api_version = 58.0
+        # Empty zip file for testing
+        self.mock_zip = generate_sample_zip_data()
+
+    # Test case for a successful deployment and deploy status
+    @patch("requests.post")
+    @patch("requests.get")
+    def test_deployment_success(self, mock_get, mock_post):
+
+        response_post = Mock(status_code=201)
+        response_post.json.return_value = {"id": "dummy_id"}
+        mock_post.return_value = response_post
+
+        response_get = Mock(status_code=200)
+        response_get.json.side_effect = [
+            {"deployResult": {"status": "InProgress"}},
+            {"deployResult": {"status": "Succeeded"}},
+        ]
+        mock_get.return_value = response_get
+
+        deployer = RestDeploy(
+            self.mock_task, self.mock_zip, False, False, "NoTestRun", []
+        )
+        deployer()
+
+        # Assertions to verify log messages
+        assert (
+            call("Deployment request successful")
+            in self.mock_logger.info.call_args_list
+        )
+        assert call("Deployment InProgress") in self.mock_logger.info.call_args_list
+        assert call("Deployment Succeeded") in self.mock_logger.info.call_args_list
+
+        # Assertions to verify API Calls
+        expected_get_calls = [
+            call(
+                "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true",
+                headers={"Authorization": "Bearer dummy_token"},
+            ),
+            call(
+                "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true",
+                headers={"Authorization": "Bearer dummy_token"},
+            ),
+        ]
+
+        mock_post.assert_called_once()
+        mock_get.assert_has_calls(expected_get_calls, any_order=True)
+
+    # Test case for a deployment failure
+    @patch("requests.post")
+    def test_deployment_failure(self, mock_post):
+
+        response_post = Mock(status_code=500)
+        response_post.json.return_value = {"id": "dummy_id"}
+        mock_post.return_value = response_post
+
+        deployer = RestDeploy(
+            self.mock_task, self.mock_zip, False, False, "NoTestRun", []
+        )
+        deployer()
+
+        # Assertions to verify log messages
+        assert (
+            call("Deployment request failed with status code 500")
+            in self.mock_logger.error.call_args_list
+        )
+
+        # Assertions to verify API Calls
+        mock_post.assert_called_once()
+
+    # Test for deployment success but deploy status failure
+    @patch("requests.post")
+    @patch("requests.get")
+    def test_deployStatus_failure(self, mock_get, mock_post):
+
+        response_post = Mock(status_code=201)
+        response_post.json.return_value = {"id": "dummy_id"}
+        mock_post.return_value = response_post
+
+        response_get = Mock(status_code=200)
+        response_get.json.side_effect = [
+            {"deployResult": {"status": "InProgress"}},
+            {
+                "deployResult": {
+                    "status": "Failed",
+                    "details": {
+                        "componentFailures": [
+                            {
+                                "problemType": "Error",
+                                "fileName": "metadata/classes/mockfile1.cls",
+                                "problem": "someproblem1",
+                                "lineNumber": 1,
+                                "columnNumber": 1,
+                            },
+                            {
+                                "problemType": "Error",
+                                "fileName": "metadata/objects/mockfile2.obj",
+                                "problem": "someproblem2",
+                                "lineNumber": 2,
+                                "columnNumber": 2,
+                            },
+                        ]
+                    },
+                }
+            },
+        ]
+        mock_get.return_value = response_get
+
+        deployer = RestDeploy(
+            self.mock_task, self.mock_zip, False, False, "NoTestRun", []
+        )
+        deployer()
+
+        # Assertions to verify log messages
+        assert (
+            call("Deployment request successful")
+            in self.mock_logger.info.call_args_list
+        )
+        assert call("Deployment InProgress") in self.mock_logger.info.call_args_list
+        assert call("Deployment Failed") in self.mock_logger.info.call_args_list
+        assert (
+            call("ERROR in file classes/mockfile1.cls: someproblem1 at line 1:1")
+            in self.mock_logger.error.call_args_list
+        )
+        assert (
+            call("ERROR in file objects/mockfile2.obj: someproblem2 at line 2:2")
+            in self.mock_logger.error.call_args_list
+        )
+
+        # Assertions to verify API Calls
+        expected_get_calls = [
+            call(
+                "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true",
+                headers={"Authorization": "Bearer dummy_token"},
+            ),
+            call(
+                "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true",
+                headers={"Authorization": "Bearer dummy_token"},
+            ),
+        ]
+
+        mock_post.assert_called_once()
+        mock_get.assert_has_calls(expected_get_calls, any_order=True)
+
+    # Test case for a deployment with a pending status
+    @patch("requests.post")
+    @patch("requests.get")
+    def test_pending_call(self, mock_get, mock_post):
+
+        response_post = Mock(status_code=201)
+        response_post.json.return_value = {"id": "dummy_id"}
+        mock_post.return_value = response_post
+
+        response_get = Mock(status_code=200)
+        response_get.json.side_effect = [
+            {"deployResult": {"status": "InProgress"}},
+            {"deployResult": {"status": "Pending"}},
+            {"deployResult": {"status": "Succeeded"}},
+        ]
+        mock_get.return_value = response_get
+
+        deployer = RestDeploy(
+            self.mock_task, self.mock_zip, False, False, "NoTestRun", []
+        )
+        deployer()
+
+        # Assertions to verify log messages
+        assert (
+            call("Deployment request successful")
+            in self.mock_logger.info.call_args_list
+        )
+        assert call("Deployment InProgress") in self.mock_logger.info.call_args_list
+        assert call("Deployment Pending") in self.mock_logger.info.call_args_list
+        assert call("Deployment Succeeded") in self.mock_logger.info.call_args_list
+
+        # Assertions to verify API Calls
+        expected_get_calls = [
+            call(
+                "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true",
+                headers={"Authorization": "Bearer dummy_token"},
+            ),
+            call(
+                "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true",
+                headers={"Authorization": "Bearer dummy_token"},
+            ),
+            call(
+                "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true",
+                headers={"Authorization": "Bearer dummy_token"},
+            ),
+        ]
+
+        mock_post.assert_called_once()
+        mock_get.assert_has_calls(expected_get_calls, any_order=True)
+
+    def test_reformat_zip(self):
+        input_zip = generate_sample_zip_data()
+        expected_zip = generate_sample_zip_data("metadata/")
+
+        deployer = RestDeploy(
+            self.mock_task, self.mock_zip, False, False, "NoTestRun", []
+        )
+        actual_output_zip = deployer._reformat_zip(input_zip)
+
+        self.assertEqual(
+            base64.b64encode(actual_output_zip).decode("utf-8"), expected_zip
+        )
+
+    def test_purge_on_delete(self):
+        test_data = [
+            ("not_sandbox_developer", "Not Developer Edition", False, False, "false"),
+            ("purgeOnDelete_true", "Developer Edition", True, True, "true"),
+            ("purgeOnDelete_none", "Developer Edition", True, None, "true"),
+        ]
+
+        for name, org_type, is_sandbox, purge_on_delete, expected_result in test_data:
+            with self.subTest(name=name):
+                self.mock_task.org_config.org_type = org_type
+                self.mock_task.org_config.is_sandbox = is_sandbox
+                deployer = RestDeploy(
+                    self.mock_task,
+                    self.mock_zip,
+                    purge_on_delete,
+                    False,
+                    "NoTestRun",
+                    [],
+                )
+                self.assertEqual(deployer.purge_on_delete, expected_result)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/cumulusci/tasks/salesforce/tests/test_Deploy.py b/cumulusci/tasks/salesforce/tests/test_Deploy.py
index a08211f11..d971f3a23 100644
--- a/cumulusci/tasks/salesforce/tests/test_Deploy.py
+++ b/cumulusci/tasks/salesforce/tests/test_Deploy.py
@@ -15,7 +15,8 @@ from .util import create_task
 
 
 class TestDeploy:
-    def test_get_api(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_get_api(self, rest_deploy):
         with temporary_dir() as path:
             touch("package.xml")
             task = create_task(
@@ -26,6 +27,7 @@ class TestDeploy:
                     "namespace_inject": "ns",
                     "namespace_strip": "ns",
                     "unmanaged": True,
+                    "rest_deply": rest_deploy,
                 },
             )
 
@@ -34,11 +36,18 @@ class TestDeploy:
             assert "package.xml" in zf.namelist()
             zf.close()
 
-    def test_get_api__managed(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_get_api__managed(self, rest_deploy):
         with temporary_dir() as path:
             touch("package.xml")
             task = create_task(
-                Deploy, {"path": path, "namespace_inject": "ns", "unmanaged": False}
+                Deploy,
+                {
+                    "path": path,
+                    "namespace_inject": "ns",
+                    "unmanaged": False,
+                    "rest_deploy": rest_deploy,
+                },
             )
 
             api = task._get_api()
@@ -46,7 +55,8 @@ class TestDeploy:
             assert "package.xml" in zf.namelist()
             zf.close()
 
-    def test_get_api__additional_options(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_get_api__additional_options(self, rest_deploy):
         with temporary_dir() as path:
             touch("package.xml")
             task = create_task(
@@ -56,6 +66,7 @@ class TestDeploy:
                     "test_level": "RunSpecifiedTests",
                     "specified_tests": "TestA,TestB",
                     "unmanaged": False,
+                    "rest_deploy": rest_deploy,
                 },
             )
 
@@ -63,7 +74,8 @@ class TestDeploy:
             assert api.run_tests == ["TestA", "TestB"]
             assert api.test_level == "RunSpecifiedTests"
 
-    def test_get_api__skip_clean_meta_xml(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_get_api__skip_clean_meta_xml(self, rest_deploy):
         with temporary_dir() as path:
             touch("package.xml")
             task = create_task(
@@ -72,6 +84,7 @@ class TestDeploy:
                     "path": path,
                     "clean_meta_xml": False,
                     "unmanaged": True,
+                    "rest_deploy": rest_deploy,
                 },
             )
 
@@ -80,7 +93,8 @@ class TestDeploy:
             assert "package.xml" in zf.namelist()
             zf.close()
 
-    def test_get_api__static_resources(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_get_api__static_resources(self, rest_deploy):
         with temporary_dir() as path:
             with open("package.xml", "w") as f:
                 f.write(
@@ -107,6 +121,7 @@ class TestDeploy:
                         "namespace_inject": "ns",
                         "namespace_strip": "ns",
                         "unmanaged": True,
+                        "rest_deploy": rest_deploy,
                     },
                 )
 
@@ -120,32 +135,37 @@ class TestDeploy:
                 assert "<members>TestBundle</members>" in package_xml
                 zf.close()
 
-    def test_get_api__missing_path(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_get_api__missing_path(self, rest_deploy):
         task = create_task(
             Deploy,
             {
                 "path": "BOGUS",
                 "unmanaged": True,
+                "rest_deploy": rest_deploy,
             },
         )
 
         api = task._get_api()
         assert api is None
 
-    def test_get_api__empty_package_zip(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_get_api__empty_package_zip(self, rest_deploy):
         with temporary_dir() as path:
             task = create_task(
                 Deploy,
                 {
                     "path": path,
                     "unmanaged": True,
+                    "rest_deploy": rest_deploy,
                 },
             )
 
             api = task._get_api()
             assert api is None
 
-    def test_init_options(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_init_options(self, rest_deploy):
         with pytest.raises(TaskOptionsError):
             create_task(
                 Deploy,
@@ -153,6 +173,7 @@ class TestDeploy:
                     "path": "empty",
                     "test_level": "RunSpecifiedTests",
                     "unmanaged": False,
+                    "rest_deploy": rest_deploy,
                 },
             )
 
@@ -169,34 +190,40 @@ class TestDeploy:
                     "test_level": "RunLocalTests",
                     "specified_tests": ["TestA"],
                     "unmanaged": False,
+                    "rest_deploy": rest_deploy,
                 },
             )
 
-    def test_init_options__transforms(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_init_options__transforms(self, rest_deploy):
         d = create_task(
             Deploy,
             {
                 "path": "src",
                 "transforms": ["clean_meta_xml"],
+                "rest_deploy": rest_deploy,
             },
         )
 
         assert len(d.transforms) == 1
         assert isinstance(d.transforms[0], CleanMetaXMLTransform)
 
-    def test_init_options__bad_transforms(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_init_options__bad_transforms(self, rest_deploy):
         with pytest.raises(TaskOptionsError) as e:
             create_task(
                 Deploy,
                 {
                     "path": "src",
                     "transforms": [{}],
+                    "rest_deploy": rest_deploy,
                 },
             )
 
             assert "transform spec is not valid" in str(e)
 
-    def test_freeze_sets_kind(self):
+    @pytest.mark.parametrize("rest_deploy", [True, False])
+    def test_freeze_sets_kind(self, rest_deploy):
         task = create_task(
             Deploy,
             {
@@ -204,6 +231,7 @@ class TestDeploy:
                 "namespace_tokenize": "ns",
                 "namespace_inject": "ns",
                 "namespace_strip": "ns",
+                "rest_deploy": rest_deploy,
             },
         )
         step = StepSpec(