Repository: SFDO-Tooling/CumulusCI
Pull request #3384 (2022-10-06 17:44:14)

Auto-load dataset matching org shape name
If `load_dataset` is called without any path options, it will automatically use a dataset that matches the org shape, if one exists. For example, a `dev` org will automatically use a dataset that exists at `datasets/dev/`. Within that folder, two files must exist, also matching the org shape name: `dev.mapping.yml` and `dev.dataset.sql`, in this example. If the directory or files do not exist and no paths options were specified, the task will look for `datasets/mapping.yml` and `datasets/dataset.sql` by default. When the `org_shape_match_only` option is `True`, this overrides any path options and default files and looks _only_ for a dataset directory that matches the org shape name. The `org_shape_match_only` option defaults to `False`.

In addition, `load_dataset` is included in `config_dev`, `config_qa`, and `config_managed`, so it is automatically called when running most org setup flows. In this context, it runs with `org_shape_match_only` set to `True`, to avoid double loading for backwards compatibility with customer flows that are already customized to call `load_dataset`.


Patch:
diff --git a/cumulusci/cumulusci.yml b/cumulusci/cumulusci.yml
index d97addcfc..8f599dfe2 100644
--- a/cumulusci/cumulusci.yml
+++ b/cumulusci/cumulusci.yml
@@ -664,9 +664,6 @@ tasks:
     load_dataset:
         description: Load a sample dataset using the bulk API.
         class_path: cumulusci.tasks.bulkdata.LoadData
-        options:
-            mapping: "datasets/mapping.yml"
-            sql_path: "datasets/sample.sql"
         group: "Data Operations"
     load_custom_settings:
         description: Load Custom Settings specified in a YAML file to the target org
@@ -798,6 +795,10 @@ flows:
                 task: deploy_post
             2:
                 task: update_admin_profile
+            90:
+                task: load_dataset
+                options:
+                    org_shape_match_only: True
     config_managed:
         group: Post-Install Configuration
         description: Configure an org for use after the managed package has been installed.
@@ -806,6 +807,10 @@ flows:
                 task: deploy_post
             2:
                 task: update_admin_profile
+            90:
+                task: load_dataset
+                options:
+                    org_shape_match_only: True
     config_packaging:
         group: Post-Install Configuration
         description: Configure packaging org for upload after package metadata is deployed
@@ -820,6 +825,10 @@ flows:
                 task: deploy_post
             2:
                 task: update_admin_profile
+            90:
+                task: load_dataset
+                options:
+                    org_shape_match_only: True
     config_regression:
         group: Post-Install Configuration
         description: Configure an org for QA regression after the package is installed
diff --git a/cumulusci/tasks/bulkdata/load.py b/cumulusci/tasks/bulkdata/load.py
index db649b806..ca04a5560 100644
--- a/cumulusci/tasks/bulkdata/load.py
+++ b/cumulusci/tasks/bulkdata/load.py
@@ -2,6 +2,7 @@ import tempfile
 import typing as T
 from collections import defaultdict
 from contextlib import contextmanager
+from pathlib import Path
 from unittest.mock import MagicMock
 
 from sqlalchemy import Column, MetaData, Table, Unicode, create_engine, func, inspect
@@ -84,6 +85,9 @@ class LoadData(SqlAlchemyMixin, BaseSalesforceApiTask):
         "set_recently_viewed": {
             "description": "By default, the first 1000 records inserted via the Bulk API will be set as recently viewed. If fewer than 1000 records are inserted, existing objects of the same type being inserted will also be set as recently viewed.",
         },
+        "org_shape_match_only": {
+            "description": "When True, all path options are ignored and only a dataset matching the org shape name will be loaded. Defaults to False."
+        },
     }
     row_warning_limit = 10
 
@@ -93,15 +97,7 @@ class LoadData(SqlAlchemyMixin, BaseSalesforceApiTask):
         self.options["ignore_row_errors"] = process_bool_arg(
             self.options.get("ignore_row_errors") or False
         )
-        if self.options.get("database_url"):
-            # prefer database_url if it's set
-            self.options["sql_path"] = None
-        elif self.options.get("sql_path"):
-            self.options["database_url"] = None
-        else:
-            raise TaskOptionsError(
-                "You must set either the database_url or sql_path option."
-            )
+        self._init_dataset()
         self.reset_oids = self.options.get("reset_oids", True)
         self.bulk_mode = (
             self.options.get("bulk_mode") and self.options.get("bulk_mode").title()
@@ -120,7 +116,82 @@ class LoadData(SqlAlchemyMixin, BaseSalesforceApiTask):
             self.options.get("set_recently_viewed", True)
         )
 
+    def _init_dataset(self):
+        """Find the dataset paths to use with the following sequence:
+        1. If `org_shape_match_only` is True (defaults False),
+           unset any other path options that may have been supplied.
+        2. Prefer a supplied `database_url`.
+        3. If `sql_path` was supplied, but not `mapping`, use default `mapping` value.
+        4. If `mapping` was supplied, but not `sql_path`, use default `sql_path` value.
+        5. If no path options were supplied, look for a dataset matching the org shape.
+        6. If no matching dataset was found AND `org_shape_match_only` is False,
+           look for a dataset with the default `mapping` and `sql_path` values
+           (as previously defaulted in the standard library yml).
+        """
+        self.options["org_shape_match_only"] = process_bool_arg(
+            self.options.get("org_shape_match_only", False)
+        )
+        if self.options["org_shape_match_only"]:
+            self.options["mapping"] = None
+            self.options["sql_path"] = None
+            self.options["database_url"] = None
+
+        self.options.setdefault("database_url", None)
+        if self.options.get("database_url"):
+            # prefer database_url if it's set
+            self.options["sql_path"] = None
+        elif self.options.get("sql_path"):
+            self.options.setdefault("mapping", "datasets/mapping.yml")
+        elif self.options.get("mapping"):
+            self.options.setdefault("sql_path", "datasets/sample.sql")
+        elif found_dataset := (
+            self._find_matching_dataset() or self._find_default_dataset()
+        ):  # didn't get either database_url or sql_path
+            mapping_path, dataset_path = found_dataset
+            self.options["mapping"] = mapping_path
+            self.options["sql_path"] = dataset_path
+        else:
+            self.has_dataset = False
+            return
+        self.has_dataset = True
+
+    def _find_matching_dataset(self) -> T.Optional[T.Tuple[str, str]]:
+        org_shape = self.org_config.lookup("config_name") if self.org_config else None
+        if not org_shape:
+            return None  # persistent org
+        dataset_folder = f"datasets/{org_shape}"
+        if Path(dataset_folder).exists():
+            # check for dataset.sql and mapping.yml
+            mapping_path = f"{dataset_folder}/{org_shape}.mapping.yml"
+            dataset_path = f"{dataset_folder}/{org_shape}.dataset.sql"
+            if Path(mapping_path).exists() and Path(dataset_path).exists():
+                return (mapping_path, dataset_path)
+            else:
+                self.logger.warning(
+                    f"Found datasets/{org_shape} but it did not contain {org_shape}.mapping.yml and {org_shape}.dataset.yml."
+                )
+        return None
+
+    def _find_default_dataset(self) -> T.Optional[T.Tuple[str, str]]:
+        if self.options["org_shape_match_only"]:
+            return None
+        dataset_path = "datasets/sample.sql"
+        mapping_path = "datasets/mapping.yml"
+        if Path(dataset_path).exists() and Path(mapping_path).exists():
+            return (mapping_path, dataset_path)
+        return None
+
     def _run_task(self):
+        if not self.has_dataset:
+            if org_shape := self.org_config.lookup("config_name"):
+                self.logger.info(
+                    f"No data will be loaded because there was no dataset found matching your org shape name ('{org_shape}')."
+                )
+            else:
+                self.logger.info(
+                    "No data will be loaded because this is a persistent org and no dataset was specified."
+                )
+            return
         self._init_mapping()
         with self._init_db():
             self._expand_mapping()
diff --git a/docs/data.md b/docs/data.md
index 5eb25761d..70764a811 100644
--- a/docs/data.md
+++ b/docs/data.md
@@ -21,6 +21,10 @@ Datasets are stored in the `datasets/` folder within a repository by
 default. Projects created with a recent version of CumulusCI ship with
 this directory in place.
 
+If `load_dataset` is called without any path options, it will automatically use a dataset that matches the org shape, if one exists. For example, a `dev` org will automatically use a dataset that exists at `datasets/dev/`. Within that folder, two files must exist, also matching the org shape name: `dev.mapping.yml` and `dev.dataset.sql`, in this example. If the directory or files do not exist and no paths options were specified, the task will look for `datasets/mapping.yml` and `datasets/dataset.sql` by default. When the `org_shape_match_only` option is `True`, this overrides any path options and default files and looks _only_ for a dataset directory that matches the org shape name. The `org_shape_match_only` option defaults to `False`.
+
+In addition, `load_dataset` is included in `config_dev`, `config_qa`, and `config_managed`, so it is automatically called when running most org setup flows. In this context, it runs with `org_shape_match_only` set to `True`, to avoid double loading for backwards compatibility with customer flows that are already customized to call `load_dataset`.
+
 ## The Lifecycle of a Dataset
 
 A dataset starts with a definition: which objects, and which fields, are


Test patch:
diff --git a/cumulusci/tasks/bulkdata/tests/test_load.py b/cumulusci/tasks/bulkdata/tests/test_load.py
index 13759454d..23ec22c54 100644
--- a/cumulusci/tasks/bulkdata/tests/test_load.py
+++ b/cumulusci/tasks/bulkdata/tests/test_load.py
@@ -1,5 +1,6 @@
 import io
 import json
+import logging
 import os
 import random
 import shutil
@@ -33,12 +34,35 @@ from cumulusci.tasks.bulkdata.tests.utils import (
 )
 from cumulusci.tests.util import (
     CURRENT_SF_API_VERSION,
+    DummyOrgConfig,
     assert_max_memory_usage,
     mock_describe_calls,
 )
 from cumulusci.utils import temporary_dir
 
 
+class FakePath:
+    def __init__(self, real_path, does_exist):
+        self.does_exist = does_exist
+        self.real_path = real_path
+
+    def exists(self):
+        return self.does_exist
+
+    def __str__(self):
+        return self.real_path
+
+
+class FakePathFileSystem:
+    def __init__(self, existing_paths):
+        self.paths = [Path(path) for path in existing_paths]
+
+    def __call__(self, *filename):
+        filename_parts = [str(el) for el in filename]
+        real_filename = str(Path(*filename_parts))
+        return FakePath(real_filename, (Path(real_filename) in self.paths))
+
+
 class TestLoadData:
     mapping_file = "mapping_v1.yml"
 
@@ -235,8 +259,10 @@ class TestLoadData:
         ]
 
     def test_init_options__missing_input(self):
-        with pytest.raises(TaskOptionsError):
-            _make_task(LoadData, {"options": {}})
+        t = _make_task(LoadData, {"options": {}})
+
+        assert t.options["sql_path"] == "datasets/sample.sql"
+        assert t.options["mapping"] == "datasets/mapping.yml"
 
     def test_init_options__bulk_mode(self):
         t = _make_task(
@@ -271,14 +297,234 @@ class TestLoadData:
 
         assert t.options["database_url"] == "file:///test.db"
         assert t.options["sql_path"] is None
+        assert t.has_dataset is True
 
-    def test_init_options__sql_path(self):
+    def test_init_options__sql_path_and_mapping(self):
         t = _make_task(
-            LoadData, {"options": {"sql_path": "test.sql", "mapping": "mapping.yml"}}
+            LoadData,
+            {
+                "options": {
+                    "sql_path": "datasets/test.sql",
+                    "mapping": "datasets/test.yml",
+                }
+            },
         )
 
-        assert t.options["sql_path"] == "test.sql"
+        assert t.options["sql_path"] == "datasets/test.sql"
+        assert t.options["mapping"] == "datasets/test.yml"
+        assert t.options["database_url"] is None
+        assert t.has_dataset is True
+
+    def test_init_options__org_shape_match_only__true(self):
+        dataset_path = "datasets/dev/dev.dataset.sql"
+        mapping_path = "datasets/dev/dev.mapping.yml"
+        fake_paths = FakePathFileSystem([mapping_path, dataset_path, "datasets/dev"])
+        with mock.patch("cumulusci.tasks.bulkdata.load.Path", fake_paths):
+            org_config = DummyOrgConfig(
+                {
+                    "config_name": "dev",
+                },
+                "test",
+            )
+            t = _make_task(
+                LoadData,
+                {
+                    "options": {
+                        "sql_path": "datasets/test.sql",
+                        "mapping": "datasets/mapping.yml",
+                        "org_shape_match_only": True,
+                    }
+                },
+                org_config,
+            )
+
+            assert t.options["sql_path"] == dataset_path
+            assert t.options["mapping"] == mapping_path
+            assert t.options["database_url"] is None
+            assert t.has_dataset is True
+
+    def test_init_options__org_shape_match_only__false(self):
+        """Matching paths exist but we do not use them."""
+        dataset_path = "datasets/dev/dev.dataset.sql"
+        mapping_path = "datasets/dev/dev.mapping.yml"
+        fake_paths = FakePathFileSystem([mapping_path, dataset_path, "datasets/dev"])
+        with mock.patch("cumulusci.tasks.bulkdata.load.Path", fake_paths):
+            org_config = DummyOrgConfig(
+                {
+                    "config_name": "dev",
+                },
+                "test",
+            )
+            t = _make_task(
+                LoadData,
+                {
+                    "options": {
+                        "sql_path": "datasets/test.sql",
+                        "mapping": "datasets/mapping.yml",
+                        #  "org_shape_match_only": False, DEFAULTED
+                    }
+                },
+                org_config,
+            )
+
+            assert t.options["sql_path"] == "datasets/test.sql"
+            assert t.options["mapping"] == "datasets/mapping.yml"
+            assert t.options["database_url"] is None
+            assert t.has_dataset is True
+
+    def test_init_options__sql_path_no_mapping(self):
+        t = _make_task(LoadData, {"options": {"sql_path": "datasets/test.sql"}})
+
+        assert t.options["sql_path"] == "datasets/test.sql"
+        assert t.options["mapping"] == "datasets/mapping.yml"
+        assert t.options["database_url"] is None
+        assert t.has_dataset is True
+
+    def test_init_options__mapping_no_sql_path(self):
+        t = _make_task(LoadData, {"options": {"mapping": "datasets/test.yml"}})
+
+        assert t.options["sql_path"] == "datasets/sample.sql"
+        assert t.options["mapping"] == "datasets/test.yml"
         assert t.options["database_url"] is None
+        assert t.has_dataset is True
+
+    def test_init_datasets__matching_dataset(self):
+        dataset_path = "datasets/dev/dev.dataset.sql"
+        mapping_path = "datasets/dev/dev.mapping.yml"
+        fake_paths = FakePathFileSystem([mapping_path, dataset_path, "datasets/dev"])
+        with mock.patch("cumulusci.tasks.bulkdata.load.Path", fake_paths):
+            org_config = DummyOrgConfig(
+                {
+                    "config_name": "dev",
+                },
+                "test",
+            )
+            t = _make_task(LoadData, {}, org_config=org_config)
+            assert t.options["sql_path"] == dataset_path
+            assert t.options["mapping"] == mapping_path
+            assert t.has_dataset is True
+
+    def test_init_datasets__matching_dataset_dir__missing_dataset_path(self, caplog):
+        caplog.set_level(logging.WARNING)
+        mapping_path = "datasets/dev/dev.mapping.yml"
+        fake_paths = FakePathFileSystem([mapping_path, "datasets/dev"])
+        org_shape = "dev"
+        with mock.patch("cumulusci.tasks.bulkdata.load.Path", fake_paths):
+            org_config = DummyOrgConfig(
+                {
+                    "config_name": org_shape,
+                },
+                "test",
+            )
+            t = _make_task(LoadData, {}, org_config=org_config)
+            assert t.options.get("sql_path") is None
+            assert t.options.get("mapping") is None
+            assert (
+                f"Found datasets/{org_shape} but it did not contain {org_shape}.mapping.yml and {org_shape}.dataset.yml."
+                in caplog.text
+            )
+            assert t.has_dataset is False
+
+    def test_init_datasets__matching_dataset_dir__missing_mapping_path(self, caplog):
+        caplog.set_level(logging.WARNING)
+        dataset_path = "datasets/dev/dev.dataset.sql"
+        fake_paths = FakePathFileSystem([dataset_path, "datasets/dev"])
+        org_shape = "dev"
+        with mock.patch("cumulusci.tasks.bulkdata.load.Path", fake_paths):
+            org_config = DummyOrgConfig(
+                {
+                    "config_name": org_shape,
+                },
+                "test",
+            )
+            t = _make_task(LoadData, {}, org_config=org_config)
+            assert t.options.get("sql_path") is None
+            assert t.options.get("mapping") is None
+            assert (
+                f"Found datasets/{org_shape} but it did not contain {org_shape}.mapping.yml and {org_shape}.dataset.yml."
+                in caplog.text
+            )
+            assert t.has_dataset is False
+
+    def test_init_datasets__no_matching_dataset__use_default(self):
+        dataset_path = "datasets/sample.sql"
+        mapping_path = "datasets/mapping.yml"
+        fake_paths = FakePathFileSystem([dataset_path, mapping_path])
+        with mock.patch("cumulusci.tasks.bulkdata.load.Path", fake_paths):
+            org_config = DummyOrgConfig(
+                {
+                    "config_name": "dev",
+                },
+                "test",
+            )
+            t = _make_task(LoadData, {}, org_config=org_config)
+            assert t.options["sql_path"] == "datasets/sample.sql"
+            assert t.options["mapping"] == "datasets/mapping.yml"
+            assert t.has_dataset is True
+
+    def test_init_datasets__no_matching_dataset__skip_default(self):
+        dataset_path = "datasets/sample.sql"
+        mapping_path = "datasets/mapping.yml"
+        fake_paths = FakePathFileSystem([dataset_path, mapping_path])
+        with mock.patch("cumulusci.tasks.bulkdata.load.Path", fake_paths):
+            org_config = DummyOrgConfig(
+                {
+                    "config_name": "dev",
+                },
+                "test",
+            )
+            t = _make_task(
+                LoadData,
+                {
+                    "options": {
+                        "org_shape_match_only": True,
+                    }
+                },
+                org_config=org_config,
+            )
+            assert t.options["sql_path"] is None
+            assert t.options["mapping"] is None
+            assert t.has_dataset is False
+
+    def test_init_datasets__no_matching_dataset_no_load_scratch(self, caplog):
+        caplog.set_level(logging.INFO)
+        fake_paths = FakePathFileSystem([])
+        with mock.patch("cumulusci.tasks.bulkdata.load.Path", fake_paths):
+            org_config = DummyOrgConfig(
+                {
+                    "config_name": "dev",
+                },
+                "test",
+            )
+            t = _make_task(LoadData, {}, org_config=org_config)
+            assert t.has_dataset is False
+            assert t.options.get("sql_path") is None
+            assert t.options.get("mapping") is None
+            t._run_task()
+            assert (
+                "No data will be loaded because there was no dataset found matching your org shape name ('dev')."
+                in caplog.text
+            )
+
+    def test_init_datasets__no_matching_dataset_no_load_persistent(self, caplog):
+        caplog.set_level(logging.INFO)
+        fake_paths = FakePathFileSystem([])
+        with mock.patch("cumulusci.tasks.bulkdata.load.Path", fake_paths):
+            org_config = DummyOrgConfig(
+                {
+                    "config_name": None,
+                },
+                "test",
+            )
+            t = _make_task(LoadData, {}, org_config=org_config)
+            assert t.has_dataset is False
+            assert t.options.get("sql_path") is None
+            assert t.options.get("mapping") is None
+            t._run_task()
+            assert (
+                "No data will be loaded because this is a persistent org and no dataset was specified."
+                in caplog.text
+            )
 
     @mock.patch("cumulusci.tasks.bulkdata.load.validate_and_inject_mapping")
     def test_init_mapping_passes_options_to_validate(self, validate_and_inject_mapping):
diff --git a/cumulusci/tasks/bulkdata/tests/utils.py b/cumulusci/tasks/bulkdata/tests/utils.py
index cab396b90..164e26a5e 100644
--- a/cumulusci/tasks/bulkdata/tests/utils.py
+++ b/cumulusci/tasks/bulkdata/tests/utils.py
@@ -8,7 +8,7 @@ from cumulusci.tasks.bulkdata.step import (
 from cumulusci.tests import util as cci_test_utils
 
 
-def _make_task(task_class, task_config):
+def _make_task(task_class, task_config, org_config=None):
     task_config = TaskConfig(task_config)
     universal_config = UniversalConfig()
     project_config = BaseProjectConfig(
@@ -22,7 +22,7 @@ def _make_task(task_class, task_config):
     )
     keychain = BaseProjectKeychain(project_config, "")
     project_config.set_keychain(keychain)
-    org_config = cci_test_utils.DummyOrgConfig(
+    org_config = org_config or cci_test_utils.DummyOrgConfig(
         {"instance_url": "https://example.com", "access_token": "abc123"}, "test"
     )
     return task_class(project_config, task_config, org_config)
diff --git a/cumulusci/tasks/tests/test_metadeploy.py b/cumulusci/tasks/tests/test_metadeploy.py
index bda320555..bd507b6fb 100644
--- a/cumulusci/tasks/tests/test_metadeploy.py
+++ b/cumulusci/tasks/tests/test_metadeploy.py
@@ -277,31 +277,31 @@ class TestPublish(GithubApiTestMixin):
 
         steps = body["steps"]
         self.maxDiff = None
+
         assert [
             {
-                "is_required": True,
-                "kind": "managed",
                 "name": "Install Test Product 1.0",
+                "kind": "managed",
+                "is_required": True,
                 "path": "install_prod.install_managed",
-                "source": None,
                 "step_num": "1/2",
                 "task_class": "cumulusci.tasks.salesforce.InstallPackageVersion",
                 "task_config": {
                     "options": {
-                        "namespace": "ns",
                         "version": "1.0",
+                        "namespace": "ns",
                         "interactive": False,
                         "base_package_url_format": "{}",
                     },
                     "checks": [],
                 },
+                "source": None,
             },
             {
-                "is_required": True,
-                "kind": "metadata",
                 "name": "Update Admin Profile",
+                "kind": "metadata",
+                "is_required": True,
                 "path": "install_prod.config_managed.update_admin_profile",
-                "source": None,
                 "step_num": "1/3/2",
                 "task_class": "cumulusci.tasks.salesforce.ProfileGrantAllAccess",
                 "task_config": {
@@ -311,6 +311,29 @@ class TestPublish(GithubApiTestMixin):
                     },
                     "checks": [],
                 },
+                "source": None,
+            },
+            {
+                "name": "load_dataset",
+                "kind": "other",
+                "is_required": True,
+                "path": "install_prod.config_managed.load_dataset",
+                "step_num": "1/3/90",
+                "task_class": "cumulusci.tasks.bulkdata.LoadData",
+                "task_config": {
+                    "options": {
+                        "org_shape_match_only": True,
+                        "ignore_row_errors": False,
+                        "mapping": None,
+                        "sql_path": None,
+                        "database_url": None,
+                        "inject_namespaces": True,
+                        "drop_missing_schema": False,
+                        "set_recently_viewed": True,
+                    },
+                    "checks": [],
+                },
+                "source": None,
             },
             {
                 "name": "util_sleep",
@@ -344,6 +367,10 @@ class TestPublish(GithubApiTestMixin):
                     "message": "Update Admin Profile",
                     "description": "title of installation step",
                 },
+                "load_dataset": {
+                    "message": "load_dataset",
+                    "description": "title of installation step",
+                },
                 "util_sleep": {
                     "message": "util_sleep",
                     "description": "title of installation step",