Repository: SFDO-Tooling/CumulusCI
Pull request #3261 (2022-06-28 16:28:09)

Extract MetaDeploy API from publish task
`make_api_session`:
- Extract the logic for setting the base URL into a subclass of
  requests.Session. This allows:
  - Read the api url and auth token directly from the ServiceConfig,
  - Automatically prepend base_url and set auth token,
  - First step in allowing calls from outside the task.

`MetaDeployApi`:
- Interface for the MetaDeploy REST API.

Tests for `metadeploy_publish` are left intact to keep them in place
as integration tests.


Patch:
diff --git a/cumulusci/core/metadeploy/api.py b/cumulusci/core/metadeploy/api.py
new file mode 100644
index 000000000..b92e24be3
--- /dev/null
+++ b/cumulusci/core/metadeploy/api.py
@@ -0,0 +1,88 @@
+"""Calls to the MetaDeploy REST API."""
+
+import functools
+from typing import Optional, Union
+
+from requests.models import Response
+from requests.sessions import Session
+
+from cumulusci.core.config import ServiceConfig
+from cumulusci.core.exceptions import CumulusCIException
+from cumulusci.utils.http.requests_utils import safe_json_from_response
+
+
+def make_api_session(metadeploy_service: ServiceConfig) -> Session:
+    """
+    Given a MetaDeploy ServiceConfig, returns a requests.Session with
+    authorization headers and service.base_url set.
+    """
+    base_url: str = metadeploy_service.url
+
+    patched_session = Session()
+    patched_session.headers["Authorization"] = f"token {metadeploy_service.token}"
+
+    def patched_request(base, func, method, url, *args, **kwargs):
+        """Call requests.request with base_url prepended."""
+        base_url = base + url.replace(base, "", 1)
+        return func(method, base_url, *args, **kwargs)
+
+    patched_session.request = functools.partial(
+        patched_request, base_url, patched_session.request
+    )
+
+    return patched_session
+
+
+class MetaDeployAPI:
+    def __init__(self, metadeploy_service: ServiceConfig) -> None:
+        self.session: Session = make_api_session(metadeploy_service=metadeploy_service)
+
+    def _create(self, obj: str, json: dict) -> dict:
+        response: Response = self.session.post(obj, json=json)
+        return safe_json_from_response(response)
+
+    def _find(self, obj: str, query: Union[dict, str]) -> Optional[dict]:
+        response: Response = self.session.get(obj, params=query)
+        result: dict = safe_json_from_response(response)
+
+        try:
+            return result["data"][0]
+        except IndexError:
+            return None
+        except KeyError:
+            raise CumulusCIException(
+                "CumulusCI received an unexpected response from MetaDeploy. "
+                "Ensure that your MetaDeploy service is configured with the Admin API URL, which "
+                "ends in /rest, and that your authentication token is valid."
+            ) from None
+
+    def create_plan(self, plan: dict) -> dict:
+        return self._create("/plans", plan)
+
+    def create_plan_template(self, template: dict) -> dict:
+        return self._create("/plantemplates", json=template)
+
+    def create_plan_slug(self, slug: dict) -> dict:
+        return self._create("/planslug", json=slug)
+
+    def create_version(self, version: dict) -> dict:
+        return self._create("/versions", json=version)
+
+    def find_product(self, repo_url: str) -> Optional[dict]:
+        return self._find("/products", {"repo_url": repo_url})
+
+    def find_plan_template(self, query: dict) -> Optional[dict]:
+        return self._find("/plantemplates", query)
+
+    def find_version(self, query: dict) -> Optional[dict]:
+        return self._find("/versions", query)
+
+    def update_version(self, pk: Union[int, str]) -> dict:
+        response: Response = self.session.patch(
+            f"/versions/{pk}", json={"is_listed": True}
+        )
+        return safe_json_from_response(response)
+
+    def update_lang_translation(self, lang: str, labels: dict) -> dict:
+        response: Response = self.session.patch(f"/translations/{lang}", json=labels)
+        return safe_json_from_response(response)
diff --git a/cumulusci/tasks/metadeploy.py b/cumulusci/tasks/metadeploy.py
index 25914b21e..9c4202578 100644
--- a/cumulusci/tasks/metadeploy.py
+++ b/cumulusci/tasks/metadeploy.py
@@ -2,7 +2,7 @@ import contextlib
 import json
 import re
 from pathlib import Path
-from typing import List, Union
+from typing import List, Optional, Union
 
 import requests
 
@@ -10,6 +10,7 @@ from cumulusci.core.config import BaseProjectConfig, FlowConfig, TaskConfig
 from cumulusci.core.exceptions import CumulusCIException, TaskOptionsError
 from cumulusci.core.flowrunner import FlowCoordinator
 from cumulusci.core.github import get_tag_by_name
+from cumulusci.core.metadeploy.api import MetaDeployAPI
 from cumulusci.core.tasks import BaseTask
 from cumulusci.core.utils import process_bool_arg
 from cumulusci.utils import cd, download_extract_github, temporary_dir
@@ -24,15 +25,13 @@ class BaseMetaDeployTask(BaseTask):
 
     def _init_task(self):
         metadeploy_service = self.project_config.keychain.get_service("metadeploy")
-        self.base_url = metadeploy_service.url
-        self.api = requests.Session()
-        self.api.headers["Authorization"] = f"token {metadeploy_service.token}"
+        self.api: MetaDeployAPI = MetaDeployAPI(metadeploy_service)
 
     def _call_api(self, method, path, collect_pages=False, **kwargs):
-        next_url = self.base_url + path
+        next_url = path
         results = []
         while next_url is not None:
-            response = self.api.request(method, next_url, **kwargs)
+            response = self.api.session.request(method, next_url, **kwargs)
             if response.status_code == 400:
                 raise requests.exceptions.HTTPError(response.content)
             response = safe_json_from_response(response)
@@ -100,6 +99,10 @@ class Publish(BaseMetaDeployTask):
 
         # Find or create Version
         product = self._find_product(repo_url)
+        if not product:
+            raise CumulusCIException(
+                f"No product found in MetaDeploy with repo URL {repo_url}"
+            )
         if not self.dry_run:
             version = self._find_or_create_version(product)
             if self.labels_path and "slug" in product:
@@ -144,11 +147,7 @@ class Publish(BaseMetaDeployTask):
 
             # Update version to set is_listed=True
             if self.publish:
-                self._call_api(
-                    "PATCH",
-                    f"/versions/{version['id']}",
-                    json={"is_listed": True},
-                )
+                self.api.update_version(version["id"])
                 self.logger.info(f"Published Version {version['url']}")
 
         # Save labels
@@ -244,7 +243,7 @@ class Publish(BaseMetaDeployTask):
             plan_json["preflight_checks"] = plan_config["checks"]
 
         # Create Plan
-        plan = self._call_api("POST", "/plans", json=plan_json)
+        plan = self.api.create_plan(plan=plan_json)
         self.logger.info(f"Created Plan {plan['url']}")
 
     def _get_allowed_org_providers(self, plan_name: str) -> List[str]:
@@ -268,19 +267,7 @@ class Publish(BaseMetaDeployTask):
         return org_providers
 
     def _find_product(self, repo_url):
-        try:
-            result = self._call_api("GET", "/products", params={"repo_url": repo_url})
-            if len(result["data"]) != 1:
-                raise CumulusCIException(
-                    f"No product found in MetaDeploy with repo URL {repo_url}"
-                )
-        except KeyError as exc:
-            raise CumulusCIException(
-                "CumulusCI received an unexpected response from MetaDeploy. "
-                "Ensure that your MetaDeploy service is configured with the Admin API URL, which "
-                "ends in /rest, and that your authentication token is valid."
-            ) from exc
-        product = result["data"][0]
+        product = self.api.find_product(repo_url)
         self._add_labels(
             product,
             "product",
@@ -300,56 +287,47 @@ class Publish(BaseMetaDeployTask):
             label = self.project_config.get_version_for_tag(self.tag)
         else:
             label = self.commit
-        result = self._call_api(
-            "GET", "/versions", params={"product": product["id"], "label": label}
+
+        version: Optional[dict] = self.api.find_version(
+            query={"product": product["id"], "label": label}
         )
-        if len(result["data"]) == 0:
-            version = self._call_api(
-                "POST",
-                "/versions",
-                json={
-                    "product": product["url"],
-                    "label": label,
-                    "description": self.options.get("description", ""),
-                    "is_production": True,
-                    "commit_ish": self.tag or self.commit,
-                    "is_listed": False,
-                },
-            )
-            self.logger.info(f"Created {version['url']}")
-        else:
-            version = result["data"][0]
-            self.logger.info(f"Found {version['url']}")
+        verb: str = "Found" if version else "Created"
+
+        if not version:
+            version_dict = {
+                "product": product["url"],
+                "label": label,
+                "description": self.options.get("description", ""),
+                "is_production": True,
+                "commit_ish": self.tag or self.commit,
+                "is_listed": False,
+            }
+            version = self.api.create_version(version_dict)
+
+        self.logger.info(f"{verb} {version['url']}")
         return version
 
     def _find_or_create_plan_template(self, product, plan_name, plan_config):
-        result = self._call_api(
-            "GET",
-            "/plantemplates",
-            params={"product": product["id"], "name": plan_name},
+        plantemplate = self.api.find_plan_template(
+            {"product": product["id"], "name": plan_name}
         )
-        if len(result["data"]) == 0:
-            plantemplate = self._call_api(
-                "POST",
-                "/plantemplates",
-                json={
+        verb: str = "Found" if plantemplate else "Created"
+
+        if not plantemplate:
+            plantemplate = self.api.create_plan_template(
+                {
                     "name": plan_name,
                     "product": product["url"],
                     "preflight_message": plan_config.get("preflight_message", ""),
                     "post_install_message": plan_config.get("post_install_message", ""),
                     "error_message": plan_config.get("error_message", ""),
-                },
+                }
             )
-            self.logger.info(f"Created {plantemplate['url']}")
-            planslug = self._call_api(
-                "POST",
-                "/planslug",
-                json={"slug": plan_config["slug"], "parent": plantemplate["url"]},
+            planslug = self.api.create_plan_slug(
+                {"slug": plan_config["slug"], "parent": plantemplate["url"]}
             )
             self.logger.info(f"Created {planslug['url']}")
-        else:
-            plantemplate = result["data"][0]
-            self.logger.info(f"Found {plantemplate['url']}")
+        self.logger.info(f"{verb} {plantemplate['url']}")
         return plantemplate
 
     def _load_labels(self):
@@ -398,6 +376,6 @@ class Publish(BaseMetaDeployTask):
 
             self.logger.info(f"Updating {lang} translations")
             try:
-                self._call_api("PATCH", f"/translations/{lang}", json=prefixed_labels)
+                self.api.update_lang_translation(lang, prefixed_labels)
             except requests.exceptions.HTTPError as err:
                 self.logger.warning(f"Could not update {lang} translation: {err}")
diff --git a/pyproject.toml b/pyproject.toml
index d49b47c05..c02826146 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -11,6 +11,9 @@ exclude_lines = ["pragma: no cover", "@abstract", "@abc.abstract" ]
 [tool.pytest.ini_options]
 testpaths = "cumulusci"
 addopts =  "-p cumulusci.tests.pytest_plugins.pytest_typeguard -p cumulusci.tests.pytest_plugins.pytest_sf_vcr -p cumulusci.tests.pytest_plugins.pytest_sf_orgconnect"
+markers = [
+    "metadeploy: mark a test that interacts with the MetaDeploy REST API",
+]
 
 [tool.isort]
 profile = "black"


Test patch:
diff --git a/cumulusci/cli/tests/test_plan.py b/cumulusci/cli/tests/test_plan.py
index c64e2e659..8bd424a2f 100644
--- a/cumulusci/cli/tests/test_plan.py
+++ b/cumulusci/cli/tests/test_plan.py
@@ -9,6 +9,8 @@ from cumulusci.cli.runtime import CliRuntime
 from .. import plan
 from .utils import run_click_command
 
+pytestmark = pytest.mark.metadeploy
+
 
 @pytest.fixture
 def runtime():
diff --git a/cumulusci/core/metadeploy/tests/test_api.py b/cumulusci/core/metadeploy/tests/test_api.py
new file mode 100644
index 000000000..f4def5e6a
--- /dev/null
+++ b/cumulusci/core/metadeploy/tests/test_api.py
@@ -0,0 +1,276 @@
+import pytest
+import responses
+from requests.models import Response
+from requests.sessions import Session
+from responses import matchers
+
+from cumulusci.core.config import ServiceConfig
+from cumulusci.core.exceptions import CumulusCIException
+from cumulusci.core.metadeploy.api import MetaDeployAPI, make_api_session
+
+pytestmark = pytest.mark.metadeploy
+DEFAULT_REST_URL: str = "https://metadeploy.example.com/api/rest"
+DEFAULT_TOKEN: str = "37b003aee9bdd744a6618e9fe12"
+
+
+@pytest.fixture
+def default_api_client():
+    return MetaDeployAPI(
+        metadeploy_service=ServiceConfig(
+            {"url": DEFAULT_REST_URL, "token": DEFAULT_TOKEN}
+        )
+    )
+
+
+@pytest.fixture
+def auth_header_matcher():
+    return matchers.header_matcher({"Authorization": f"token {DEFAULT_TOKEN}"})
+
+
+@responses.activate
+def test_make_api(auth_header_matcher):
+    expected_result: dict = {"data": [1]}
+    responses.add(
+        "GET",
+        f"{DEFAULT_REST_URL}/resource",
+        json=expected_result,
+        match=[auth_header_matcher],
+    )
+    responses.add(
+        "POST",
+        f"{DEFAULT_REST_URL}/resource",
+        json=expected_result,
+        match=[auth_header_matcher],
+    )
+    service: ServiceConfig = ServiceConfig(
+        {"url": f"{DEFAULT_REST_URL}", "token": DEFAULT_TOKEN}
+    )
+    api_session: Session = make_api_session(service)
+    resp: Response = api_session.get("/resource")
+    result: dict = resp.json()
+    assert result == expected_result
+    _ = api_session.get(f"{DEFAULT_REST_URL}/resource")
+
+    assert responses.assert_call_count(f"{DEFAULT_REST_URL}/resource", 2) is True
+
+
+def test_find_product(default_api_client, auth_header_matcher):
+    params = {"repo_url": "https://api.github.com/repos/TestOwner/TestRepo"}
+    payload = {
+        "data": [
+            {
+                "id": "abcdef",
+                "url": "https://EXISTING_PRODUCT",
+                "slug": "existing",
+            }
+        ]
+    }
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.GET,
+            url=f"{DEFAULT_REST_URL}/products",
+            match=(auth_header_matcher, matchers.query_param_matcher(params)),
+            json=payload,
+        )
+
+        product: dict = default_api_client.find_product(params["repo_url"])
+    assert product == payload["data"][0]
+
+
+def test_find_product__api_err(default_api_client, auth_header_matcher):
+    params = {"repo_url": "https://api.github.com/repos/TestOwner/MISSING"}
+    payload = {"result": []}
+
+    with pytest.raises(CumulusCIException, match="unexpected response"):
+        with responses.RequestsMock() as rsps:
+            rsps.add(
+                method=responses.GET,
+                url=f"{DEFAULT_REST_URL}/products",
+                match=(auth_header_matcher, matchers.query_param_matcher(params)),
+                json=payload,
+            )
+
+            default_api_client.find_product(params["repo_url"])
+
+
+def test_create_plan(default_api_client, auth_header_matcher):
+    payload = {"url": f"{DEFAULT_REST_URL}/plans/1"}
+    plan = {
+        "is_listed": True,
+        "plan_template": "https://foo.example.com/1",
+        "post_install_message_additional": "",
+        "preflight_message_additional": "",
+        "steps": [],
+        "supported_orgs": "persistent",
+        "tier": "primary",
+        "title": "Install Tubulator 1.0",
+        "version": f"{DEFAULT_REST_URL}/version/1",
+        "visible_to": None,
+    }
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.POST,
+            url=f"{DEFAULT_REST_URL}/plans",
+            match=(auth_header_matcher, matchers.json_params_matcher(plan)),
+            status=201,
+            json=payload,
+        )
+        result = default_api_client.create_plan(plan)
+    assert result == payload
+
+
+def test_find_version(default_api_client, auth_header_matcher):
+    query = {"product": "bDcaVz", "label": "1.0"}
+    payload = {"data": [{"url": "http://EXISTING_VERSION"}]}
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.GET,
+            url=f"{DEFAULT_REST_URL}/versions",
+            match=(auth_header_matcher, matchers.query_param_matcher(query)),
+            json=payload,
+        )
+
+        result = default_api_client.find_version(query)
+
+    assert result == payload["data"][0]
+
+
+def test_find_version_missing_none(default_api_client, auth_header_matcher):
+    query = {"product": "bDcaVz", "label": "1.0"}
+    payload = {"data": []}
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.GET,
+            url=f"{DEFAULT_REST_URL}/versions",
+            match=(auth_header_matcher, matchers.query_param_matcher(query)),
+            json=payload,
+        )
+
+        result = default_api_client.find_version(query)
+
+    assert result is None
+
+
+def test_create_version(default_api_client, auth_header_matcher):
+    version = {
+        "product": f"{DEFAULT_REST_URL}/product/foo",
+        "label": "1.0",
+        "description": "",
+        "is_production": True,
+        "commit_ish": "tag_or_sha",
+        "is_listed": False,
+    }
+    payload = {"url": f"http://{DEFAULT_REST_URL}/versions/1", "id": 1}
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.POST,
+            url=f"{DEFAULT_REST_URL}/versions",
+            match=(auth_header_matcher, matchers.json_params_matcher(version)),
+            status=201,
+            json=payload,
+        )
+
+        version = default_api_client.create_version(version)
+
+    assert version == payload
+
+
+def test_publish_version(default_api_client, auth_header_matcher):
+    vers = {"is_listed": True}
+    pk = "XG4pVvD"
+    payload = {"url": f"http://{DEFAULT_REST_URL}/versions/{pk}", "id": pk}
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.PATCH,
+            url=f"{DEFAULT_REST_URL}/versions/{pk}",
+            match=(auth_header_matcher, matchers.json_params_matcher(vers)),
+            json=payload,
+        )
+
+        updated_version = default_api_client.update_version(pk)
+    assert payload == updated_version
+
+
+def test_find_plan_template(default_api_client, auth_header_matcher):
+    query = {"product": "bDcaVz", "name": "Shinra"}
+    payload = {"data": [{"url": "http://existing_template"}]}
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.GET,
+            url=f"{DEFAULT_REST_URL}/plantemplates",
+            match=(auth_header_matcher, matchers.query_param_matcher(query)),
+            json=payload,
+        )
+
+        template = default_api_client.find_plan_template(query)
+
+    assert template == payload["data"][0]
+
+
+def test_create_plan_template(default_api_client, auth_header_matcher):
+    template = {
+        "name": "plan_name",
+        "product": "product",
+        "preflight_message": "",
+        "post_install_message": "",
+        "error_message": "oops",
+    }
+    payload = {"url": "http://EXISTING_VERSION"}
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.POST,
+            url=f"{DEFAULT_REST_URL}/plantemplates",
+            match=(auth_header_matcher, matchers.json_params_matcher(template)),
+            status=201,
+            json=payload,
+        )
+
+        created_template = default_api_client.create_plan_template(template)
+
+    assert created_template == payload
+
+
+def test_create_plan_slug(default_api_client, auth_header_matcher):
+    slug = {"slug": "plan_name", "parent": "product"}
+    payload = {"url": "http://EXISTING_VERSION"}
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.POST,
+            url=f"{DEFAULT_REST_URL}/planslug",
+            match=(auth_header_matcher, matchers.json_params_matcher(slug)),
+            status=201,
+            json=payload,
+        )
+
+        created_slug = default_api_client.create_plan_slug(slug)
+
+    assert created_slug == payload
+
+
+def test_publish_labels(default_api_client, auth_header_matcher):
+    lang = "en_us"
+    labels = {
+        "title": "name of product",
+        "error_message": "shown after failed installation (markdown)",
+    }
+    payload = {}
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.PATCH,
+            url=f"{DEFAULT_REST_URL}/translations/{lang}",
+            match=(auth_header_matcher, matchers.json_params_matcher(labels)),
+            json=payload,
+        )
+
+        updated_txn = default_api_client.update_lang_translation(lang, labels)
+    assert payload == updated_txn
diff --git a/cumulusci/tasks/tests/test_metadeploy.py b/cumulusci/tasks/tests/test_metadeploy.py
index 77e1c31ca..bda320555 100644
--- a/cumulusci/tasks/tests/test_metadeploy.py
+++ b/cumulusci/tasks/tests/test_metadeploy.py
@@ -17,6 +17,8 @@ from cumulusci.tasks.github.tests.util_github_api import GithubApiTestMixin
 from cumulusci.tasks.metadeploy import BaseMetaDeployTask, Publish
 from cumulusci.tests.util import create_project_config
 
+pytestmark = pytest.mark.metadeploy
+
 
 class TestBaseMetaDeployTask:
     maxDiff = None