Repository: SFDO-Tooling/CumulusCI
Pull request #3270 (2022-07-07 17:23:39)

 CumulusCI now supports pinning dependencies on other projects
CumulusCI now supports pinning dependencies on other projects. This capability is particularly important for ISVs extending NPSP and EDA. See the documentation for details on how to create dependency pins.


Patch:
diff --git a/cumulusci/cli/project.py b/cumulusci/cli/project.py
index a194b3c5a..2302e6444 100644
--- a/cumulusci/cli/project.py
+++ b/cumulusci/cli/project.py
@@ -359,7 +359,8 @@ def project_info(runtime):
 @pass_runtime(require_keychain=True)
 def project_dependencies(runtime, resolution_strategy):
     dependencies = get_static_dependencies(
-        runtime.project_config, resolution_strategy=resolution_strategy
+        runtime.project_config,
+        resolution_strategy=resolution_strategy,
     )
     for line in dependencies:
         click.echo(f"{line}")
diff --git a/cumulusci/core/dependencies/dependencies.py b/cumulusci/core/dependencies/dependencies.py
index 170294b69..fe1c8a985 100644
--- a/cumulusci/core/dependencies/dependencies.py
+++ b/cumulusci/core/dependencies/dependencies.py
@@ -65,6 +65,44 @@ def _validate_github_parameters(values):
     return values
 
 
+class DependencyPin(HashableBaseModel, abc.ABC):
+    @abc.abstractmethod
+    def can_pin(self, d: "DynamicDependency") -> bool:
+        ...
+
+    @abc.abstractmethod
+    def pin(self, d: "DynamicDependency", context: BaseProjectConfig):
+        ...
+
+
+DependencyPin.update_forward_refs()
+
+
+class GitHubDependencyPin(DependencyPin):
+    """Model representing a request to pin a GitHub dependency to a specific tag"""
+
+    github: str
+    tag: str
+
+    def can_pin(self, d: "DynamicDependency") -> bool:
+        return isinstance(d, BaseGitHubDependency) and d.github == self.github
+
+    def pin(self, d: "BaseGitHubDependency", context: BaseProjectConfig):
+        from cumulusci.core.dependencies.resolvers import (  # Circular imports
+            GitHubTagResolver,
+        )
+
+        if d.tag and d.tag != self.tag:
+            raise DependencyResolutionError(
+                f"A pin is specified for {self.github}, but the dependency already has a tag specified."
+            )
+        d.tag = self.tag
+        d.ref, d.package_dependency = GitHubTagResolver().resolve(d, context)
+
+
+GitHubDependencyPin.update_forward_refs()
+
+
 class Dependency(HashableBaseModel, abc.ABC):
     """Abstract base class for models representing dependencies
 
@@ -130,17 +168,30 @@ class DynamicDependency(Dependency, abc.ABC):
     def is_flattened(self):
         return False
 
-    def resolve(self, context, strategies):
+    def resolve(
+        self,
+        context: BaseProjectConfig,
+        strategies: List,  # List[DependencyResolutionStrategy], but circular import
+        pins: Optional[List[DependencyPin]] = None,
+    ):
         """Resolve a DynamicDependency that is not pinned to a specific version into one that is."""
         # avoid import cycle
         from .resolvers import resolve_dependency
 
+        for pin in pins or []:
+            if pin.can_pin(self):
+                context.logger.info(f"Pinning dependency {self} to {pin}")
+                pin.pin(self, context)
+                return
+
         resolve_dependency(self, context, strategies)
 
 
 class BaseGitHubDependency(DynamicDependency, abc.ABC):
     """Base class for dynamic dependencies that reference a GitHub repo."""
 
+    pin_class = GitHubDependencyPin
+
     github: Optional[AnyUrl] = None
 
     repo_owner: Optional[str] = None  # Deprecated - use full URL
@@ -658,6 +709,39 @@ class UnmanagedZipURLDependency(UnmanagedDependency):
         return f"{self.zip_url} {subfolder}"
 
 
+def parse_pins(pins: Optional[List[dict]]) -> List[DependencyPin]:
+    """Convert a list of dependency pin specifications in the form of dicts
+    (as defined in `cumulusci.yml`) and parse each into a concrete DependencyPin subclass.
+
+    Throws DependencyParseError if a dict cannot be parsed."""
+    parsed_pins = []
+    for pin in pins or []:
+        parsed = parse_dependency_pin(pin)
+        if parsed is None:
+            raise DependencyParseError(f"Unable to parse dependency pin: {pin}")
+        parsed_pins.append(parsed)
+
+    return parsed_pins
+
+
+AVAILABLE_DEPENDENCY_PIN_CLASSES = [GitHubDependencyPin]
+
+
+def parse_dependency_pin(pin_dict: dict) -> Optional[DependencyPin]:
+    """Parse a single dependency pin specification in the form of a dict
+    into a concrete DependencyPin subclass.
+
+    Returns None if the given dict cannot be parsed."""
+
+    for dependency_pin_class in AVAILABLE_DEPENDENCY_PIN_CLASSES:
+        try:
+            pin = dependency_pin_class.parse_obj(pin_dict)
+            if pin:
+                return pin
+        except pydantic.ValidationError:
+            pass
+
+
 def parse_dependencies(deps: Optional[List[dict]]) -> List[Dependency]:
     """Convert a list of dependency specifications in the form of dicts
     (as defined in `cumulusci.yml`) and parse each into a concrete Dependency subclass.
@@ -672,6 +756,16 @@ def parse_dependencies(deps: Optional[List[dict]]) -> List[Dependency]:
     return parsed_deps
 
 
+AVAILABLE_DEPENDENCY_CLASSES = [
+    PackageVersionIdDependency,
+    PackageNamespaceVersionDependency,
+    UnmanagedGitHubRefDependency,
+    UnmanagedZipURLDependency,
+    GitHubDynamicDependency,
+    GitHubDynamicSubfolderDependency,
+]
+
+
 def parse_dependency(dep_dict: dict) -> Optional[Dependency]:
     """Parse a single dependency specification in the form of a dict
     into a concrete Dependency subclass.
@@ -685,14 +779,7 @@ def parse_dependency(dep_dict: dict) -> Optional[Dependency]:
     # We also want PackageVersionIdDependency to match before
     # PackageNamespaceVersionDependency, which can also accept a `version_id`.
 
-    for dependency_class in [
-        PackageVersionIdDependency,
-        PackageNamespaceVersionDependency,
-        UnmanagedGitHubRefDependency,
-        UnmanagedZipURLDependency,
-        GitHubDynamicDependency,
-        GitHubDynamicSubfolderDependency,
-    ]:
+    for dependency_class in AVAILABLE_DEPENDENCY_CLASSES:
         try:
             dep = dependency_class.parse_obj(dep_dict)
             if dep:
diff --git a/cumulusci/core/dependencies/resolvers.py b/cumulusci/core/dependencies/resolvers.py
index e913bb273..448839527 100644
--- a/cumulusci/core/dependencies/resolvers.py
+++ b/cumulusci/core/dependencies/resolvers.py
@@ -9,11 +9,13 @@ from cumulusci.core.config.project_config import BaseProjectConfig
 from cumulusci.core.dependencies.dependencies import (
     BaseGitHubDependency,
     Dependency,
+    DependencyPin,
     DynamicDependency,
     PackageNamespaceVersionDependency,
     PackageVersionIdDependency,
     StaticDependency,
     parse_dependencies,
+    parse_pins,
 )
 from cumulusci.core.dependencies.github import (
     get_package_data,
@@ -474,10 +476,11 @@ def dependency_filter_ignore_deps(ignore_deps: List[dict]) -> Callable:
 
 def get_static_dependencies(
     context: BaseProjectConfig,
-    dependencies: List[Dependency] = None,
-    resolution_strategy: str = None,
-    strategies: List[DependencyResolutionStrategy] = None,
-    filter_function: Callable = None,
+    dependencies: Optional[List[Dependency]] = None,
+    resolution_strategy: Optional[str] = None,
+    strategies: Optional[List[DependencyResolutionStrategy]] = None,
+    filter_function: Optional[Callable] = None,
+    pins: Optional[List[DependencyPin]] = None,
 ) -> List[StaticDependency]:
     """Resolves the dependencies of a CumulusCI project
     to convert dynamic GitHub dependencies into static dependencies
@@ -496,6 +499,8 @@ def get_static_dependencies(
     """
     if dependencies is None:
         dependencies = parse_dependencies(context.project__dependencies)
+    if pins is None:
+        pins = parse_pins(context.lookup("project__dependency_pins"))
     assert (resolution_strategy is None and strategies is not None) or (
         strategies is None and resolution_strategy is not None
     ), "Expected resolution_strategy or strategies but not both"
@@ -507,7 +512,8 @@ def get_static_dependencies(
     while any(not d.is_flattened or not d.is_resolved for d in dependencies):
         for d in dependencies:
             if isinstance(d, DynamicDependency) and not d.is_resolved:
-                d.resolve(context, strategies)
+                # Finish resolving the dependency using our given strategies.
+                d.resolve(context, strategies, pins)
 
         def unique(it: Iterable):
             seen = set()
@@ -528,13 +534,15 @@ def get_static_dependencies(
         )
 
     # Make sure, if we had no flattening or resolving to do, that we apply the ignore list.
-    return [d for d in dependencies if filter_function(d)]
+    # Type is guaranteed via the logic above.
+    return [d for d in dependencies if filter_function(d)]  # type: ignore
 
 
 def resolve_dependency(
     dependency: DynamicDependency,
     context: BaseProjectConfig,
     strategies: List[DependencyResolutionStrategy],
+    pins: Optional[List[DependencyPin]] = None,
 ):
     """Resolve a DynamicDependency that is not pinned to a specific version into one that is.
 
diff --git a/cumulusci/schema/cumulusci.jsonschema.json b/cumulusci/schema/cumulusci.jsonschema.json
index c5588fe7e..14de6ea72 100644
--- a/cumulusci/schema/cumulusci.jsonschema.json
+++ b/cumulusci/schema/cumulusci.jsonschema.json
@@ -383,6 +383,16 @@
                 "dependency_resolutions": {
                     "$ref": "#/definitions/DependencyResolutions"
                 },
+                "dependency_pins": {
+                    "title": "Dependency Pins",
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "additionalProperties": {
+                            "type": "string"
+                        }
+                    }
+                },
                 "source_format": {
                     "title": "Source Format",
                     "default": "mdapi",
diff --git a/cumulusci/tasks/github/release.py b/cumulusci/tasks/github/release.py
index 8983260b9..dcbca6274 100644
--- a/cumulusci/tasks/github/release.py
+++ b/cumulusci/tasks/github/release.py
@@ -76,7 +76,7 @@ class CreateRelease(BaseGithubTask):
             self.project_config,
             dependencies=parse_dependencies(
                 self.options.get("dependencies")
-                or self.project_config.project__dependencies
+                or self.project_config.project__dependencies,
             ),
             resolution_strategy=self.options.get("resolution_strategy") or "production",
         )
diff --git a/cumulusci/tasks/metaxml.py b/cumulusci/tasks/metaxml.py
index 7b466292f..04c93c36d 100644
--- a/cumulusci/tasks/metaxml.py
+++ b/cumulusci/tasks/metaxml.py
@@ -59,7 +59,8 @@ class UpdateDependencies(MetaXmlBaseTask):
     def _init_task(self):
         self.dependencies = []
         dependencies = get_static_dependencies(
-            self.project_config, resolution_strategy="production"
+            self.project_config,
+            resolution_strategy="production",
         )
         self._process_dependencies(dependencies)
 
diff --git a/cumulusci/utils/yaml/cumulusci_yml.py b/cumulusci/utils/yaml/cumulusci_yml.py
index 4038fa841..1bcbd5996 100644
--- a/cumulusci/utils/yaml/cumulusci_yml.py
+++ b/cumulusci/utils/yaml/cumulusci_yml.py
@@ -133,14 +133,15 @@ class DependencyResolutions(CCIDictModel):
 
 
 class Project(CCIDictModel):
-    name: str = None
-    package: Package = None
-    test: Test = None
-    git: Git = None
-    dependencies: List[Dict[str, str]] = None  # TODO
-    dependency_resolutions: DependencyResolutions = None
+    name: Optional[str] = None
+    package: Optional[Package] = None
+    test: Optional[Test] = None
+    git: Optional[Git] = None
+    dependencies: Optional[List[Dict[str, str]]] = None
+    dependency_resolutions: Optional[DependencyResolutions] = None
+    dependency_pins: Optional[List[Dict[str, str]]]
     source_format: Literal["sfdx", "mdapi"] = "mdapi"
-    custom: Dict = None
+    custom: Optional[Dict] = None
 
 
 class ScratchOrg(CCIDictModel):
diff --git a/docs/dev.md b/docs/dev.md
index e88c6695e..70b7691f1 100644
--- a/docs/dev.md
+++ b/docs/dev.md
@@ -273,7 +273,7 @@ Now `config_qa` (which is included in the `qa_org` flow) executes the
 ## Manage Dependencies
 
 CumulusCI is built to automate the complexities of dependency management
-for projects that extend and implement managed packages. CumulusCI
+for projects that extend, customize, or compose other projects. CumulusCI
 currently handles these main types of dependencies for projects.
 
 -   **GitHub Repository**: Dynamically resolve a product release, and
@@ -495,6 +495,64 @@ before deploying the unpackaged dependency.
 
 For more on this topic see [](namespace-injection).
 
+(pinning-github-dependencies)=
+
+### Pinning GitHub Dependencies
+
+By default, CumulusCI resolves dynamic GitHub dependencies to the latest
+available releases. In some cases, this may be undesirable. You can use
+dependency pinning to control how dependencies are resolved, including
+transitive dependencies referenced by your own direct dependencies.
+
+Use the `project__dependency_pins` section of your `cumulusci.yml` to
+establish pins. Each pin includes the keys `github`, which must match
+the URL of the repo you wish to pin, and a `tag` to which you wish to
+pin the dependency. Here's an example that pins NPSP and its transitive
+dependencies to specific versions:
+
+```yaml
+project:
+    dependencies:
+        - github: https://github.com/SalesforceFoundation/NPSP
+    dependency_pins:
+        - github: https://github.com/SalesforceFoundation/NPSP
+          tag: rel/3.219
+        - github: https://github.com/SalesforceFoundation/Contacts_and_Organizations
+          tag: rel/3.19
+        - github: https://github.com/SalesforceFoundation/Households
+          tag: rel/3.16
+        - github: https://github.com/SalesforceFoundation/Recurring_Donations
+          tag: rel/3.22
+        - github: https://github.com/SalesforceFoundation/Relationships
+          tag: rel/3.12
+        - github: https://github.com/SalesforceFoundation/Affiliations
+          tag: rel/3.10
+```
+
+Pins affect resolution of managed package versions and any unmanaged dependencies
+included in the target repositories.
+
+If CumulusCI encounters a conflict with an existing tag or other specifier
+while attempting to pin dependencies, like this:
+
+```yaml
+project:
+    dependencies:
+        - github: https://github.com/SalesforceFoundation/NPSP
+          tag: rel/3.220
+    dependency_pins:
+        - github: https://github.com/SalesforceFoundation/NPSP
+          tag: rel/3.219
+```
+
+it will stop and require you to resolve the conflict by removing either the pin
+or the dependency specification.
+
+We recommend using pins only when referencing external products whose development
+process or release schedule you do not control, such as NPSP and EDA.
+In most cases, it's preferable for dependencies within a product suite to remain
+unpinned to support ongoing development.
+
 (controlling-github-dependency-resolution)=
 
 ### Controlling GitHub Dependency Resolution


Test patch:
diff --git a/cumulusci/core/dependencies/tests/conftest.py b/cumulusci/core/dependencies/tests/conftest.py
index e40513e38..d3021d66a 100644
--- a/cumulusci/core/dependencies/tests/conftest.py
+++ b/cumulusci/core/dependencies/tests/conftest.py
@@ -38,6 +38,7 @@ def github():
         },
         [
             DummyRelease("release/2.0", "2.0"),
+            DummyRelease("release/1.5", "1.5"),
         ],
     )
 
@@ -62,6 +63,7 @@ def github():
         },
         [
             DummyRelease("release/1.1", "1.1"),
+            DummyRelease("release/1.0", "1.0"),
         ],
     )
 
@@ -317,6 +319,7 @@ def github():
 @pytest.fixture
 def project_config(github):
     pc = mock.Mock()
+    pc.lookup.return_value = None
 
     def get_repo_from_url(url):
         return github.repository(*split_repo_url(url))
diff --git a/cumulusci/core/dependencies/tests/test_resolvers.py b/cumulusci/core/dependencies/tests/test_resolvers.py
index de61e977d..dc8203df4 100644
--- a/cumulusci/core/dependencies/tests/test_resolvers.py
+++ b/cumulusci/core/dependencies/tests/test_resolvers.py
@@ -9,6 +9,7 @@ from cumulusci.core.config.project_config import BaseProjectConfig
 from cumulusci.core.config.tests.test_config import DummyRelease
 from cumulusci.core.dependencies.dependencies import (
     DynamicDependency,
+    GitHubDependencyPin,
     GitHubDynamicDependency,
     PackageNamespaceVersionDependency,
     PackageVersionIdDependency,
@@ -634,6 +635,113 @@ class TestStaticDependencyResolution:
             ),
         ]
 
+    def test_get_static_dependencies__pins(self, project_config):
+        gh = GitHubDynamicDependency(github="https://github.com/SFDO-Tooling/RootRepo")
+
+        tag = mock.Mock()
+        tag.return_value.object.sha = "tag_sha"
+        tag.return_value.message = """
+package_type: 1GP
+
+version_id: 04t000000000000"""
+        project_config.get_repo_from_url(
+            "https://github.com/SFDO-Tooling/RootRepo"
+        ).tag = tag
+
+        project_config.get_repo_from_url(
+            "https://github.com/SFDO-Tooling/DependencyRepo"
+        ).tag = tag
+
+        # Add a pin for the direct dependency and a transitive dependency
+        pins = [
+            GitHubDependencyPin(
+                github="https://github.com/SFDO-Tooling/DependencyRepo",
+                tag="release/1.0",
+            ),
+            GitHubDependencyPin(
+                github="https://github.com/SFDO-Tooling/RootRepo", tag="release/1.5"
+            ),
+        ]
+
+        assert pins[1].can_pin(gh)
+
+        deps = get_static_dependencies(
+            project_config,
+            dependencies=[gh],
+            strategies=[DependencyResolutionStrategy.RELEASE_TAG],
+            pins=pins,
+        )
+
+        assert deps == [
+            UnmanagedGitHubRefDependency(
+                github="https://github.com/SFDO-Tooling/DependencyRepo",
+                subfolder="unpackaged/pre/top",
+                unmanaged=True,
+                ref="tag_sha",
+            ),
+            PackageNamespaceVersionDependency(
+                namespace="foo",
+                version="1.0",  # from the pinned tag
+                package_name="DependencyRepo",
+                version_id="04t000000000000",
+            ),
+            UnmanagedGitHubRefDependency(
+                github="https://github.com/SFDO-Tooling/DependencyRepo",
+                subfolder="unpackaged/post/top",
+                unmanaged=False,
+                ref="tag_sha",
+                namespace_inject="foo",
+            ),
+            UnmanagedGitHubRefDependency(
+                github="https://github.com/SFDO-Tooling/RootRepo",
+                subfolder="unpackaged/pre/first",
+                unmanaged=True,
+                ref="tag_sha",
+            ),
+            UnmanagedGitHubRefDependency(
+                github="https://github.com/SFDO-Tooling/RootRepo",
+                subfolder="unpackaged/pre/second",
+                unmanaged=True,
+                ref="tag_sha",
+            ),
+            PackageNamespaceVersionDependency(
+                namespace="bar",
+                version="1.5",  # From pinned tag
+                package_name="RootRepo",
+                version_id="04t000000000000",
+            ),
+            UnmanagedGitHubRefDependency(
+                github="https://github.com/SFDO-Tooling/RootRepo",
+                subfolder="unpackaged/post/first",
+                unmanaged=False,
+                ref="tag_sha",
+                namespace_inject="bar",
+            ),
+        ]
+
+    def test_get_static_dependencies__conflicting_pin(self, project_config):
+        gh = GitHubDynamicDependency(
+            github="https://github.com/SFDO-Tooling/RootRepo", tag="release/foo"
+        )
+
+        # Add a pin for the direct dependency that conflicts
+        pins = [
+            GitHubDependencyPin(
+                github="https://github.com/SFDO-Tooling/RootRepo", tag="release/1.5"
+            ),
+        ]
+
+        assert pins[0].can_pin(gh)
+        with pytest.raises(
+            DependencyResolutionError, match="dependency already has a tag specified"
+        ):
+            get_static_dependencies(
+                project_config,
+                dependencies=[gh],
+                strategies=[DependencyResolutionStrategy.RELEASE_TAG],
+                pins=pins,
+            )
+
     def test_get_static_dependencies__ignore_namespace(self, project_config):
         gh = GitHubDynamicDependency(github="https://github.com/SFDO-Tooling/RootRepo")