Repository: SFDO-Tooling/CumulusCI
Pull request #3265 (2022-06-30 21:40:06)

Filtering for Schema caching and querying
It is now possible for a consumer of the Schema querying/caching feature to specify criteria such as "I'm only interested in  deepClonable objects" or "I'm not interested in objects that start with the word Macro" or "I'm only interested in objects populated with data" and so forth.



Patch:
diff --git a/cumulusci/salesforce_api/filterable_objects.py b/cumulusci/salesforce_api/filterable_objects.py
new file mode 100644
index 000000000..037d8cec3
--- /dev/null
+++ b/cumulusci/salesforce_api/filterable_objects.py
@@ -0,0 +1,261 @@
+NOT_COUNTABLE = (
+    "ContentDocumentLink",  # ContentDocumentLink requires a filter by a single Id on ContentDocumentId or LinkedEntityId
+    "ContentFolderItem",  # Implementation restriction: ContentFolderItem requires a filter by Id or ParentContentFolderId
+    "ContentFolder",  # Similar to above
+    "ContentFolderLink",  # Similar to above
+    "ContentFolderMember",  # Similar to above
+    "IdeaComment",  # you must filter using the following syntax: CommunityId = [single ID],
+    "Vote",  # you must filter using the following syntax: ParentId = [single ID],
+    "RecordActionHistory",  # Gack: 1133111327-118855 (1126216936)
+)
+
+
+NOT_EXTRACTABLE = NOT_COUNTABLE + (
+    "%Share",
+    "AccountBrandShare",
+    "AccountShare",
+    "AccountUserTerritory2View",
+    "ActionPlanShare",
+    "ActionPlanTemplateShare",
+    "AgentWorkShare",
+    "AnalysisSetupStatusEvent",
+    "ApptBundleConfigShare",
+    "ApptBundlePolicyShare",
+    "AssetShare",
+    "AuthorizationFormConsentShare",
+    "AuthorizationFormDataUse",
+    "AuthorizationFormDataUseHistory",
+    "AuthorizationFormDataUseShare",
+    "AuthorizationFormShare",
+    "BudgetShare",
+    "BusinessBrandShare",
+    "CalendarViewShare",
+    "CampaignShare",
+    "CaseShare",
+    "ChangeRequestShare",
+    "ChannelProgramLevelShare",
+    "ChannelProgramMemberShare",
+    "ChannelProgramShare",
+    "CommSubscriptionChannelTypeShare",
+    "CommSubscriptionConsentShare",
+    "CommSubscriptionShare",
+    "ContactPointAddressShare",
+    "ContactPointConsentShare",
+    "ContactPointEmailShare",
+    "ContactPointPhoneShare",
+    "ContactPointTypeConsentShare",
+    "ContactRequestShare",
+    "ContactShare",
+    "ContentUserSubscription",
+    "ContentWorkspacePermission",
+    "CustomHelpMenuSection",
+    "CustomObjectUserLicenseMetrics",
+    "CustomPermission",
+    "CustomPermissionDependency",
+    "CustomerShare",
+    "DataIntegrationRecordPurchasePermission",
+    "DataPrepServiceLocatorShare",
+    "DataUseLegalBasis",
+    "DataUseLegalBasisHistory",
+    "DataUseLegalBasisShare",
+    "DataUsePurpose",
+    "DataUsePurposeHistory",
+    "DataUsePurposeShare",
+    "DatasetAccess",
+    "DelegatedAccountShare",
+    "DialerCallUsageShare",
+    "DocumentChecklistItemShare",
+    "EngagementChannelTypeShare",
+    "ExpenseReportShare",
+    "ExpenseShare",
+    "ExternalDataUserAuth",
+    "ExternalEventMappingShare",
+    "FieldPermissions",
+    "FlowInterviewLogShare",
+    "FlowInterviewShare",
+    "FlowTestResultShare",
+    "ForecastingShare",
+    "ForecastingUserPreference",
+    "Group",
+    "GroupMember",
+    "ImageShare",
+    "IncidentShare",
+    "IndividualShare",
+    "JobProfileShare",
+    "KnowledgeableUser",
+    "LeadShare",
+    "ListEmailShare",
+    "LiveAgentSessionShare",
+    "LiveChatObjectAccessConfig",
+    "LiveChatObjectAccessDefinition",
+    "LiveChatTranscriptShare",
+    "LiveChatUserConfig",
+    "LiveChatUserConfigProfile",
+    "LiveChatUserConfigUser",
+    "LocationShare",
+    "LocationTrustMeasureShare",
+    "MacroShare",
+    "MacroUsageShare",
+    "MaintenancePlanShare",
+    "MaintenanceWorkRuleShare",
+    "MessagingEndUser",
+    "MessagingEndUserHistory",
+    "MessagingEndUserShare",
+    "MessagingSessionShare",
+    "MutingPermissionSet",
+    "NetworkUserHistoryRecent",
+    "ObjectPermissions",
+    "OmniSupervisorConfigUser",
+    "OpportunityShare",
+    "OrderShare",
+    "OutgoingEmail",
+    "OutgoingEmailRelation",
+    "PartnerFundAllocationShare",
+    "PartnerFundClaimShare",
+    "PartnerFundRequestShare",
+    "PartnerMarketingBudgetShare",
+    "PartyConsentShare",
+    "PendingServiceRoutingShare",
+    "PermissionSet",
+    "PermissionSetAssignment",
+    "PermissionSetGroup",
+    "PermissionSetGroupComponent",
+    "PermissionSetLicense",
+    "PermissionSetLicenseAssign",
+    "PermissionSetTabSetting",
+    "PortalDelegablePermissionSet",
+    "PresenceUserConfig",
+    "PresenceUserConfigProfile",
+    "PresenceUserConfigUser",
+    "PricebookShare",
+    "ProblemShare",
+    "ProcessExceptionShare",
+    "ProductItemShare",
+    "ProductRequestShare",
+    "ProductServiceCampaignShare",
+    "ProductTransferShare",
+    "ProfileSkillShare",
+    "ProfileSkillUser",
+    "ProfileSkillUserFeed",
+    "ProfileSkillUserHistory",
+    "PromptActionShare",
+    "PromptErrorShare",
+    "QuickTextShare",
+    "QuickTextUsageShare",
+    "QuoteShare",
+    "RecordsetFilterCriteriaShare",
+    "RecurrenceScheduleShare",
+    "ReturnOrderShare",
+    "SOSSessionShare",
+    "SchedulingConstraintShare",
+    "ScorecardShare",
+    "SellerShare",
+    "ServiceAppointmentShare",
+    "ServiceContractShare",
+    "ServiceCrewShare",
+    "ServiceResourcePreferenceShare",
+    "ServiceResourceShare",
+    "ServiceTerritoryShare",
+    "SetupEntityAccess",
+    "SharingRecordCollectionShare",
+    "SharingUserCoverage",
+    "ShiftPatternShare",
+    "ShiftShare",
+    "ShiftTemplateShare",
+    "ShipmentShare",
+    "SkillUser",
+    "SocialPostShare",
+    "StreamingChannelShare",
+    "SurveyEngagementContextShare",
+    "SurveyInvitationShare",
+    "SurveyShare",
+    "TimeSheetShare",
+    "TodayGoalShare",
+    "TopicUserEvent",
+    "TranslationUser",
+    "TravelModeShare",
+    "User",
+    "UserAccountTeamMember",
+    "UserAppInfo",
+    "UserAppMenuCustomization",
+    "UserAppMenuCustomizationShare",
+    "UserAppMenuItem",
+    "UserChangeEvent",
+    "UserConfigTransferButton",
+    "UserConfigTransferSkill",
+    "UserCustomBadge",
+    "UserCustomBadgeLocalization",
+    "UserEmailPreferredPerson",
+    "UserEmailPreferredPersonShare",
+    "UserEntityAccess",
+    "UserFeed",
+    "UserFieldAccess",
+    "UserLicense",
+    "UserListView",
+    "UserListViewCriterion",
+    "UserLogin",
+    "UserNavItem",
+    "UserPackageLicense",
+    "UserPermissionAccess",
+    "UserPreference",
+    "UserProvAccount",
+    "UserProvAccountStaging",
+    "UserProvMockTarget",
+    "UserProvisioningConfig",
+    "UserProvisioningLog",
+    "UserProvisioningRequest",
+    "UserProvisioningRequestShare",
+    "UserRecordAccess",
+    "UserRole",
+    "UserServicePresence",
+    "UserServicePresenceShare",
+    "UserSetupAppInfo",
+    "UserSetupEntityAccess",
+    "UserShare",
+    "UserTeamMember",
+    "UserTerritory2Association",
+    "VideoCallShare",
+    "VisualforceAccessMetrics",
+    "VoiceCallShare",
+    "WarrantyTermShare",
+    "WorkAccess",
+    "WorkAccessShare",
+    "WorkBadgeDefinitionShare",
+    "WorkOrderShare",
+    "WorkPlanSelectionRuleShare",
+    "WorkPlanShare",
+    "WorkPlanTemplateShare",
+    "WorkStepTemplateShare",
+    "WorkThanksShare",
+    "WorkTypeGroupShare",
+    "WorkTypeShare",
+)
+
+# Generated with these patterns:
+#     "%permission%",
+#     "%use%",
+#     "%access%",
+#     "group",
+#     "%share",
+#     "NetworkUserHistoryRecent",
+#     "ObjectPermissions",
+#     "OmniSupervisorConfigUser",
+#     "OpportunityShare",
+#     "OrderShare",
+#     "OutgoingEmail",
+#     "OutgoingEmailRelation",
+# )
+
+# And this code:
+#
+# patterns_to_ignore = NOT_EXTRACTABLE_WITHOUT_NOT_COUNTABLE
+
+# names = [obj["name"] for obj in sf.describe()["sobjects"]]
+
+# regexps_to_ignore = [
+#     re.compile(pat.replace("%", ".*"), re.IGNORECASE) for pat in patterns_to_ignore
+# ]
+
+# for name in names:
+#     if any(reg.match(name) for reg in regexps_to_ignore) and not ("__" in name):
+#         print(name)
diff --git a/cumulusci/salesforce_api/org_schema.py b/cumulusci/salesforce_api/org_schema.py
index 1de3fa44c..a78c0c75c 100644
--- a/cumulusci/salesforce_api/org_schema.py
+++ b/cumulusci/salesforce_api/org_schema.py
@@ -1,19 +1,20 @@
 import gzip
+import re
 import typing as T
 from collections import defaultdict
 from contextlib import ExitStack, contextmanager
 from email.utils import parsedate
+from enum import Enum
 from logging import getLogger
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Dict, Iterable, List, NamedTuple, Optional, Tuple
 
-from simple_salesforce import Salesforce
-from sqlalchemy import MetaData, create_engine
+from sqlalchemy import MetaData, create_engine, not_
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import create_session, exc, sessionmaker
 
-from cumulusci.core.config.org_config import OrgConfig
+from cumulusci.salesforce_api.filterable_objects import NOT_COUNTABLE, NOT_EXTRACTABLE
 from cumulusci.salesforce_api.org_schema_models import (
     Base,
     Field,
@@ -24,6 +25,7 @@ from cumulusci.utils.http.multi_request import (
     RECOVERABLE_ERRORS,
     CompositeParallelSalesforce,
 )
+from cumulusci.utils.salesforce.count_sobjects import count_sobjects
 
 y2k = "Sat, 1 Jan 2000 00:00:01 GMT"
 
@@ -44,10 +46,68 @@ def unzip_database(gzipfile, outfile):
                 db.write(gzipped.read())
 
 
+def ignore_based_on_name(objname, patterns: T.Sequence[re.Pattern]):
+    return any(pat.fullmatch(objname) for pat in patterns)
+
+
+class Filters(Enum):
+    """Options for filtering schemas"""
+
+    # These were originally done using SQL Alchemy syntax because filtering
+    # was done in SQL Alchemy. It is still possible to use them that way
+    # in client code:
+    #
+    # schema.objects.filter(Filters.activateable)
+    activateable = SObject.activateable
+    compactLayoutable = SObject.compactLayoutable
+    createable = SObject.createable
+    deepCloneable = SObject.deepCloneable
+    deletable = SObject.deletable
+    layoutable = SObject.layoutable
+    listviewable = SObject.listviewable
+    lookupLayoutable = SObject.lookupLayoutable
+    mergeable = SObject.mergeable
+    queryable = SObject.queryable
+    replicateable = SObject.replicateable
+    retrieveable = SObject.retrieveable
+    searchLayoutable = SObject.searchLayoutable
+    searchable = SObject.searchable
+    triggerable = SObject.triggerable
+    undeletable = SObject.undeletable
+    updateable = SObject.updateable
+
+    not_activateable = not_(SObject.activateable)
+    not_compactLayoutable = not_(SObject.compactLayoutable)
+    not_createable = not_(SObject.createable)
+    not_deepCloneable = not_(SObject.deepCloneable)
+    not_deletable = not_(SObject.deletable)
+    not_layoutable = not_(SObject.layoutable)
+    not_listviewable = not_(SObject.listviewable)
+    not_lookupLayoutable = not_(SObject.lookupLayoutable)
+    not_mergeable = not_(SObject.mergeable)
+    not_queryable = not_(SObject.queryable)
+    not_replicateable = not_(SObject.replicateable)
+    not_retrieveable = not_(SObject.retrieveable)
+    not_searchLayoutable = not_(SObject.searchLayoutable)
+    not_searchable = not_(SObject.searchable)
+    not_triggerable = not_(SObject.triggerable)
+    not_undeletable = not_(SObject.undeletable)
+    not_updateable = not_(SObject.updateable)
+
+    extractable = "extractable"  # can it be extracted safely?
+    # non-extractable objects are discovered
+    # through experimentation.
+    # Also, all non-queryable and non-retrievable objects are
+    # considered non-extractable.
+
+    populated = SObject.count > 0  # does it have data in the org?
+
+
 class Schema:
     """Represents an org's schema, cached from describe() calls"""
 
     _last_modified_date = None
+    included_objects = None
 
     def __init__(self, engine, schema_path):
         self.engine = engine
@@ -57,25 +117,28 @@ class Schema:
 
     @property
     def sobjects(self):
-        return self.session.query(SObject)
+        query = self.session.query(SObject)
+        if self.included_objects:
+            query = query.filter(SObject.name.in_(self.included_objects))
+        return query
 
     def __getitem__(self, name):
         try:
-            return self.session.query(SObject).filter_by(name=name).one()
+            return self.sobjects.filter_by(name=name).one()
         except exc.NoResultFound:
             raise KeyError(f"No sobject named {name}")
 
     def __contains__(self, name):
-        return self.session.query(SObject).filter_by(name=name).all()
+        return self.sobjects.filter_by(name=name).all()
 
     def keys(self):
-        return (x[0] for x in self.session.query(SObject.name).all())
+        return [x.name for x in self.sobjects.all()]
 
     def values(self):
-        return self.session.query(SObject).all()
+        return self.sobjects.all()
 
     def items(self):
-        return ((obj.name, obj) for obj in self.session.query(SObject).all())
+        return [(obj.name, obj) for obj in self.sobjects]
 
     def block_writing(self):
         """After this method is called, the database can't be updated again"""
@@ -108,15 +171,38 @@ class Schema:
     def __repr__(self):
         return f"<Schema {self.path} : {self.engine}>"
 
-    def populate_cache(self, sf, last_modified_date, include=None, logger=None):
+    def populate_cache(
+        self,
+        sf,
+        included_objects,
+        last_modified_date,
+        filters: T.Sequence[Filters] = (),
+        patterns_to_ignore: T.Sequence[str] = (),
+        logger=None,
+    ) -> T.List[str]:
         """Populate a schema cache from the API, using last_modified_date
         to pull down only new schema"""
+        for pat in patterns_to_ignore:
+            assert pat.replace("%", "").isidentifier(), f"Pattern has wrong chars {pat}"
+
+        # Platform bug!
+        patterns_to_ignore = ("MacroInstruction%",) + tuple(patterns_to_ignore)
+        regexps_to_ignore = [
+            re.compile(pat.replace("%", ".*"), re.IGNORECASE)
+            for pat in patterns_to_ignore
+        ]
 
-        if include:
-            sobj_names = include
-        else:
-            sobjs = sf.describe()["sobjects"]
-            sobj_names = [obj["name"] for obj in sobjs]
+        objs = [obj for obj in sf.describe()["sobjects"]]
+        if included_objects:
+            objs = [obj for obj in objs if obj["name"] in included_objects]
+        sobj_names = [
+            obj["name"]
+            for obj in objs
+            if not (
+                ignore_based_on_name(obj["name"], regexps_to_ignore)
+                or ignore_based_on_properties(obj, filters)
+            )
+        ]
 
         responses = list(deep_describe(sf, last_modified_date, sobj_names, logger))
         changes = [
@@ -125,17 +211,16 @@ class Schema:
             if resp.status == 200
         ]
         unexpected = [resp for resp in responses if resp.status not in (200, 304)]
-        for unknown in unexpected:
+        for unknown in unexpected:  # pragma: no cover
             logger.warning(
                 f"Unexpected describe reply. An SObject may be missing: {unknown}"
             )
         self._populate_cache_from_describe(changes, last_modified_date)
+        return sobj_names
 
     def _populate_cache_from_describe(
-        self,
-        describe_objs: List[Tuple[dict, str]],
-        last_modified_date,
-    ):
+        self, describe_objs: List[Tuple[dict, str]], last_modified_date
+    ) -> T.List[str]:
         """Populate a schema cache from a list of describe objects."""
         engine = self.engine
         metadata = Base.metadata
@@ -229,20 +314,40 @@ class BufferedSession:
 
 @contextmanager
 def get_org_schema(
-    sf: Salesforce,
-    org_config: OrgConfig,
-    include: T.Sequence[str] = None,
+    sf,
+    org_config,
     *,
+    include_counts: bool = False,
+    filters: T.Sequence[Filters] = (),
+    patterns_to_ignore: T.Tuple[str] = (),
+    included_objects: T.List[str] = (),
     force_recache=False,
     logger=None,
 ):
     """
     Get a read-only representation of an org's schema.
-
+    sf - simple_saleforce object
     org_config - an OrgConfig for the relevant org
+
+    include_counts: query each queryable/retrievable object count.
+                    This takes time and may even timeout in huge orgs!
+    filters: A sequence of Filters which are the same as Salesforce SObject properties
+            like .createable, .deletable etc. Objects that do not match are ignored.
+            Two special filters exist:
+                * Filters.extractable, which uses heuristics to limit to objects
+                    that extract properly
+                * Filters.populated, which limits to objects that have data in them.
+                    This depends upon `include_counts`
+    included_objects: Ignore objects not in this list. Stacks with other filters
+    patterns_to_ignore: Strings in SQL %LIKE% syntax that match SObjects to be
+                        ignored.
+
     force_recache: True - replace cache. False (default) - use/update cache is available.
     logger - replace the standard logger "cumulusci.salesforce_api.org_schema"
     """
+    assert not isinstance(patterns_to_ignore, str)
+
+    filters = set(filters)
     with org_config.get_orginfo_cache_dir(Schema.__module__) as directory:
         directory.mkdir(exist_ok=True, parents=True)
         schema_path = directory / "org_schema.db.gz"
@@ -250,6 +355,18 @@ def get_org_schema(
         if force_recache and schema_path.exists():
             schema_path.unlink()
 
+        if Filters.populated in filters:
+            filters.add(Filters.queryable)
+            filters.add(Filters.retrieveable)
+            # experiment with removing this limitation by using limit 1 query instead
+            assert include_counts, "Filters.populated depends on include_counts"
+            patterns_to_ignore += NOT_COUNTABLE
+
+        if Filters.extractable in filters:
+            filters.add(Filters.queryable)
+            filters.add(Filters.retrieveable)
+            patterns_to_ignore += NOT_EXTRACTABLE
+
         logger = logger or getLogger(__name__)
 
         with ExitStack() as closer:
@@ -263,8 +380,11 @@ def get_org_schema(
                     unzip_database(schema_path, tempfile)
                     cleanups_on_failure.extend([schema_path.unlink, tempfile.unlink])
                     engine = create_engine(f"sqlite:///{str(tempfile)}")
+                    schema = Schema(
+                        engine,
+                        schema_path,
+                    )
 
-                    schema = Schema(engine, schema_path)
                     cleanups_on_failure.append(schema.close)
                     closer.callback(schema.close)
                     assert schema.sobjects.first().name
@@ -281,17 +401,50 @@ def get_org_schema(
                 engine = create_engine(f"sqlite:///{str(tempfile)}")
                 Base.metadata.bind = engine
                 Base.metadata.create_all()
-                schema = Schema(engine, schema_path)
+                schema = Schema(
+                    engine,
+                    schema_path,
+                )
                 closer.callback(schema.close)
                 schema.from_cache = False
 
-            schema.populate_cache(sf, schema.last_modified_date or y2k, include, logger)
+            objs_cached = schema.populate_cache(
+                sf,
+                included_objects,
+                schema.last_modified_date or y2k,
+                filters,
+                patterns_to_ignore,
+                logger,
+            )
+            if include_counts:
+                objs_counted = populate_counts(sf, schema, objs_cached, logger)
+
+            if Filters.populated in filters:
+                # another way to compute this might be by querying the first ID
+                objs_cached = [objname for objname, count in objs_counted if count > 0]
+            schema.included_objects = objs_cached
             schema.block_writing()
             # save a gzipped copy for later
             zip_database(tempfile, schema_path)
             yield schema
 
 
+def populate_counts(sf, schema, objs_cached, logger):
+    objects_to_count = [objname for objname in objs_cached]
+    counts, transports_errors, salesforce_errors = count_sobjects(sf, objects_to_count)
+    errors = transports_errors + salesforce_errors
+    for error in errors[0:10]:
+        logger.warning(f"Error counting SObjects: {error}")
+
+    if len(errors) > 10:
+        logger.warning(f"{len(errors)} more counting errors suppressed")
+
+    for objname, count in counts.items():
+        schema[objname].count = count
+    schema.session.flush()
+    return counts.items()
+
+
 class DescribeResponse(NamedTuple):
     """Result of a describe call from Salesforce"""
 
@@ -308,6 +461,7 @@ def deep_describe(
     Fetch describe data for sobjects from the list 'objs'
     which have changed since last_modified_date (in HTTP
     proto format) and yield each object as a DescribeResponse object."""
+
     logger = logger or getLogger(__name__)
     last_modified_date = last_modified_date or y2k
     with CompositeParallelSalesforce(sf, max_workers=8) as cpsf:
@@ -320,8 +474,6 @@ def deep_describe(
                     "httpHeaders": {"If-Modified-Since": last_modified_date},
                 }
                 for obj in objs
-                # Platform bug!!!
-                if not obj.startswith("MacroInstruction")
             )
         )
 
@@ -348,3 +500,7 @@ def deep_describe(
             for response in responses
         )
         yield from responses
+
+
+def ignore_based_on_properties(obj: dict, filters: T.Sequence[Filters]):
+    return not all(obj.get(filter.name, True) for filter in filters)
diff --git a/cumulusci/salesforce_api/org_schema_models.py b/cumulusci/salesforce_api/org_schema_models.py
index e96e1b49d..5d33b134c 100644
--- a/cumulusci/salesforce_api/org_schema_models.py
+++ b/cumulusci/salesforce_api/org_schema_models.py
@@ -125,6 +125,7 @@ class SObject(OrgSchemaModelMixin, Base):
     urls = Column(MappingType)
     supportedScopes = Column(SequenceType)
     actionOverrides = Column(SequenceType)
+    count = Column(Integer)
 
 
 field_references = Table(


Test patch:
diff --git a/cumulusci/utils/tests/cassettes/ManualEdit_test_describe_to_sql.yaml b/cumulusci/utils/tests/cassettes/ManualEdit_test_describe_to_sql.yaml
index c6ab3c314..2a0a07f9b 100644
--- a/cumulusci/utils/tests/cassettes/ManualEdit_test_describe_to_sql.yaml
+++ b/cumulusci/utils/tests/cassettes/ManualEdit_test_describe_to_sql.yaml
@@ -1,160 +1,191 @@
 interactions:
-  - request:
-      body:
-        '{"compositeRequest": [{"method": "GET", "url": "/services/data/v48.0/sobjects/Account/describe",
-        "referenceId": "refAccount", "httpHeaders": {"If-Modified-Since": "Sat, 1 Jan
-        2000 00:00:01 GMT"}}, {"method": "GET", "url": "/services/data/v48.0/sobjects/Contact/describe",
-        "referenceId": "refContact", "httpHeaders": {"If-Modified-Since": "Sat, 1 Jan
-        2000 00:00:01 GMT"}}]}'
-      headers:
-        Request-Headers:
-          - Elided
-      method: POST
-      uri: https://orgname.my.salesforce.com/services/data/v49.0/composite
-    response:
-      body:
-        string:
-          '{"compositeResponse": [{"body": {"urls": {"a":"b"}, "actionOverrides": [], "activateable":
-          false, "compactLayoutable": true, "createable": true, "custom": false, "customSetting":
-          false, "deepCloneable": false, "defaultImplementation": null, "deletable":
-          true, "deprecatedAndHidden": false, "extendedBy": null, "extendsInterfaces":
-          null, "feedEnabled": true, "fields": [{"aggregatable": true, "aiPredictionField":
-          false, "autoNumber": false, "byteLength": 18, "calculated": false, "calculatedFormula":
-          null, "cascadeDelete": false, "caseSensitive": false, "compoundFieldName":
-          null, "controllerName": null, "createable": false, "custom": false, "defaultValue":
-          null, "defaultValueFormula": null, "defaultedOnCreate": true, "dependentPicklist":
-          false, "deprecatedAndHidden": false, "digits": 0, "displayLocationInDecimal":
-          false, "encrypted": false, "externalId": false, "extraTypeInfo": null, "filterable":
-          true, "filteredLookupInfo": null, "formulaTreatNullNumberAsZero": false, "groupable":
-          true, "highScaleNumber": false, "htmlFormatted": false, "idLookup": true,
-          "inlineHelpText": null, "label": "Account ID", "length": 18, "mask": null,
-          "maskType": null, "name": "Id", "nameField": false, "namePointing": false,
-          "nillable": false, "permissionable": false, "picklistValues": [], "polymorphicForeignKey":
-          false, "precision": 0, "queryByDistance": false, "referenceTargetField": null,
-          "referenceTo": [], "relationshipName": null, "relationshipOrder": null, "restrictedDelete":
-          false, "restrictedPicklist": false, "scale": 0, "searchPrefilterable": false,
-          "soapType": "tns:ID", "sortable": true, "type": "id", "unique": false, "updateable":
-          false, "writeRequiresMasterRead": false}], "hasSubtypes": false, "implementedBy":
-          null, "implementsInterfaces": null, "isInterface": false, "isSubtype": false,
-          "keyPrefix": "001", "label": "Account", "labelPlural": "Accounts", "layoutable":
-          true, "listviewable": null, "lookupLayoutable": null, "mergeable": true, "mruEnabled":
-          true, "name": "Account", "namedLayoutInfos": [], "networkScopeFieldName":
-          null, "queryable": true, "replicateable": true, "retrieveable": true, "searchLayoutable":
-          true, "searchable": true, "sobjectDescribeOption": "FULL", "triggerable":
-          true, "undeletable": true, "updateable": true}, "httpHeaders": {"ETag": "\"baf7f695\"",
-          "Last-Modified": "Fri, 14 Aug 2020 20:53:02 GMT"}, "httpStatusCode": 200,
-          "referenceId": "refAccount"}, {"body": {"actionOverrides": [], "activateable":
-          false, "compactLayoutable": true, "createable": true, "custom": false, "customSetting":
-          false, "deepCloneable": false, "defaultImplementation": null, "deletable":
-          true, "deprecatedAndHidden": false, "extendedBy": null, "extendsInterfaces":
-          null, "feedEnabled": true, "fields": [{"aggregatable": true, "aiPredictionField":
-          false, "autoNumber": false, "byteLength": 18, "calculated": false, "calculatedFormula":
-          null, "cascadeDelete": false, "caseSensitive": false, "compoundFieldName":
-          null, "controllerName": null, "createable": false, "custom": false, "defaultValue":
-          null, "defaultValueFormula": null, "defaultedOnCreate": true, "dependentPicklist":
-          false, "deprecatedAndHidden": false, "digits": 0, "displayLocationInDecimal":
-          false, "encrypted": false, "externalId": false, "extraTypeInfo": null, "filterable":
-          true, "filteredLookupInfo": null, "formulaTreatNullNumberAsZero": false, "groupable":
-          true, "highScaleNumber": false, "htmlFormatted": false, "idLookup": true,
-          "inlineHelpText": null, "label": "Contact ID", "length": 18, "mask": null,
-          "maskType": null, "name": "Id", "nameField": false, "namePointing": false,
-          "nillable": false, "permissionable": false, "picklistValues": [], "polymorphicForeignKey":
-          false, "precision": 0, "queryByDistance": false, "referenceTargetField": null,
-          "referenceTo": [], "relationshipName": null, "relationshipOrder": null, "restrictedDelete":
-          false, "restrictedPicklist": false, "scale": 0, "searchPrefilterable": false,
-          "soapType": "tns:ID", "sortable": true, "type": "id", "unique": false, "updateable":
-          false, "writeRequiresMasterRead": false}], "hasSubtypes": false, "implementedBy":
-          null, "implementsInterfaces": null, "isInterface": false, "isSubtype": false,
-          "keyPrefix": "003", "label": "Contact", "labelPlural": "Contacts", "layoutable":
-          true, "listviewable": null, "lookupLayoutable": null, "mergeable": true, "mruEnabled":
-          true, "name": "Contact", "namedLayoutInfos": [], "networkScopeFieldName":
-          null, "queryable": true, "replicateable": true, "retrieveable": true, "searchLayoutable":
-          true, "searchable": true, "sobjectDescribeOption": "FULL", "triggerable":
-          true, "undeletable": true, "updateable": true}, "httpHeaders": {"ETag": "\"baf7f695\"",
-          "Last-Modified": "Fri, 14 Aug 2020 20:53:02 GMT"}, "httpStatusCode": 200,
-          "referenceId": "refContact"}]}'
-      headers:
-        Response-Headers: Elided
-      status:
-        code: 200
-        message: OK
-  - request:
-      body:
-        '{"compositeRequest": [{"method": "GET", "url": "/services/data/v48.0/sobjects/Opportunity/describe",
-        "referenceId": "refOpportunity", "httpHeaders": {"If-Modified-Since": "Sat,
-        1 Jan 2000 00:00:01 GMT"}}, {"method": "GET", "url": "/services/data/v48.0/sobjects/Campaign/describe",
-        "referenceId": "refCampaign", "httpHeaders": {"If-Modified-Since": "Sat, 1 Jan
-        2000 00:00:01 GMT"}}]}'
-      headers:
-        Request-Headers:
-          - Elided
-      method: POST
-      uri: https://orgname.my.salesforce.com/services/data/v49.0/composite
-    response:
-      body:
-        string:
-          '{"compositeResponse": [{"body": {"actionOverrides": [], "activateable":
-          false, "compactLayoutable": true, "createable": true, "custom": false, "customSetting":
-          false, "deepCloneable": true, "defaultImplementation": null, "deletable":
-          true, "deprecatedAndHidden": false, "extendedBy": null, "extendsInterfaces":
-          null, "feedEnabled": true, "fields": [{"aggregatable": true, "aiPredictionField":
-          false, "autoNumber": false, "byteLength": 18, "calculated": false, "calculatedFormula":
-          null, "cascadeDelete": false, "caseSensitive": false, "compoundFieldName":
-          null, "controllerName": null, "createable": false, "custom": false, "defaultValue":
-          null, "defaultValueFormula": null, "defaultedOnCreate": true, "dependentPicklist":
-          false, "deprecatedAndHidden": false, "digits": 0, "displayLocationInDecimal":
-          false, "encrypted": false, "externalId": false, "extraTypeInfo": null, "filterable":
-          true, "filteredLookupInfo": null, "formulaTreatNullNumberAsZero": false, "groupable":
-          true, "highScaleNumber": false, "htmlFormatted": false, "idLookup": true,
-          "inlineHelpText": null, "label": "Opportunity ID", "length": 18, "mask": null,
-          "maskType": null, "name": "Id", "nameField": false, "namePointing": false,
-          "nillable": false, "permissionable": false, "picklistValues": [], "polymorphicForeignKey":
-          false, "precision": 0, "queryByDistance": false, "referenceTargetField": null,
-          "referenceTo": [], "relationshipName": null, "relationshipOrder": null, "restrictedDelete":
-          false, "restrictedPicklist": false, "scale": 0, "searchPrefilterable": false,
-          "soapType": "tns:ID", "sortable": true, "type": "id", "unique": false, "updateable":
-          false, "writeRequiresMasterRead": false}], "hasSubtypes": false, "implementedBy":
-          null, "implementsInterfaces": null, "isInterface": false, "isSubtype": false,
-          "keyPrefix": "006", "label": "Opportunity", "labelPlural": "Opportunities",
-          "layoutable": true, "listviewable": null, "lookupLayoutable": null, "mergeable":
-          false, "mruEnabled": true, "name": "Opportunity", "namedLayoutInfos": [],
-          "networkScopeFieldName": null, "queryable": true, "replicateable": true, "retrieveable":
-          true, "searchLayoutable": true, "searchable": true, "sobjectDescribeOption":
-          "FULL", "triggerable": true, "undeletable": true, "updateable": true}, "httpHeaders":
-          {"ETag": "\"baf7f695\"", "Last-Modified": "Fri, 14 Aug 2020 20:53:02 GMT"},
-          "httpStatusCode": 200, "referenceId": "refOpportunity"}, {"body": {"actionOverrides":
-          [], "activateable": false, "compactLayoutable": true, "createable": true,
-          "custom": false, "customSetting": false, "deepCloneable": true, "defaultImplementation":
-          null, "deletable": true, "deprecatedAndHidden": false, "extendedBy": null,
-          "extendsInterfaces": null, "feedEnabled": false, "fields": [{"aggregatable":
-          true, "aiPredictionField": false, "autoNumber": false, "byteLength": 18, "calculated":
-          false, "calculatedFormula": null, "cascadeDelete": false, "caseSensitive":
-          false, "compoundFieldName": null, "controllerName": null, "createable": false,
-          "custom": false, "defaultValue": null, "defaultValueFormula": null, "defaultedOnCreate":
-          true, "dependentPicklist": false, "deprecatedAndHidden": false, "digits":
-          0, "displayLocationInDecimal": false, "encrypted": false, "externalId": false,
-          "extraTypeInfo": null, "filterable": true, "filteredLookupInfo": null, "formulaTreatNullNumberAsZero":
-          false, "groupable": true, "highScaleNumber": false, "htmlFormatted": false,
-          "idLookup": true, "inlineHelpText": null, "label": "Campaign ID", "length":
-          18, "mask": null, "maskType": null, "name": "Id", "nameField": false, "namePointing":
-          false, "nillable": false, "permissionable": false, "picklistValues": [], "polymorphicForeignKey":
-          false, "precision": 0, "queryByDistance": false, "referenceTargetField": null,
-          "referenceTo": [], "relationshipName": null, "relationshipOrder": null, "restrictedDelete":
-          false, "restrictedPicklist": false, "scale": 0, "searchPrefilterable": false,
-          "soapType": "tns:ID", "sortable": true, "type": "id", "unique": false, "updateable":
-          false, "writeRequiresMasterRead": false}], "hasSubtypes": false, "implementedBy":
-          null, "implementsInterfaces": null, "isInterface": false, "isSubtype": false,
-          "keyPrefix": "701", "label": "Campaign", "labelPlural": "Campaigns", "layoutable":
-          true, "listviewable": null, "lookupLayoutable": null, "mergeable": false,
-          "mruEnabled": true, "name": "Campaign", "namedLayoutInfos": [], "networkScopeFieldName":
-          null, "queryable": true, "replicateable": true, "retrieveable": true, "searchLayoutable":
-          true, "searchable": true, "sobjectDescribeOption": "FULL", "triggerable":
-          true, "undeletable": true, "updateable": true}, "httpHeaders": {"ETag": "\"baf7f695\"",
-          "Last-Modified": "Fri, 14 Aug 2020 20:53:02 GMT"}, "httpStatusCode": 200,
-          "referenceId": "refCampaign"}]}'
-      headers:
-        Response-Headers: Elided
-      status:
-        code: 200
-        message: OK
+    - request:
+          body:
+              '{"compositeRequest": [{"method": "GET", "url": "/services/data/v48.0/sobjects/Account/describe",
+              "referenceId": "refAccount", "httpHeaders": {"If-Modified-Since": "Sat, 1 Jan
+              2000 00:00:01 GMT"}}, {"method": "GET", "url": "/services/data/v48.0/sobjects/Contact/describe",
+              "referenceId": "refContact", "httpHeaders": {"If-Modified-Since": "Sat, 1 Jan
+              2000 00:00:01 GMT"}}]}, {"method": "GET", "url": "/services/data/v48.0/sobjects/PermissionSet/describe",
+              "referenceId": "refPermissionSet", "httpHeaders": {"If-Modified-Since": "Sat, 1 Jan
+              2000 00:00:01 GMT"}}]}'
+          headers:
+              Request-Headers:
+                  - Elided
+          method: POST
+          uri: https://orgname.my.salesforce.com/services/data/v49.0/composite
+      response:
+          body:
+              string:
+                  '{"compositeResponse": [{"body": {"urls": {"a":"b"}, "actionOverrides": [], "activateable":
+                  false, "compactLayoutable": true, "createable": true, "custom": false, "customSetting":
+                  false, "deepCloneable": false, "defaultImplementation": null, "deletable":
+                  true, "deprecatedAndHidden": false, "extendedBy": null, "extendsInterfaces":
+                  null, "feedEnabled": true, "fields": [{"aggregatable": true, "aiPredictionField":
+                  false, "autoNumber": false, "byteLength": 18, "calculated": false, "calculatedFormula":
+                  null, "cascadeDelete": false, "caseSensitive": false, "compoundFieldName":
+                  null, "controllerName": null, "createable": false, "custom": false, "defaultValue":
+                  null, "defaultValueFormula": null, "defaultedOnCreate": true, "dependentPicklist":
+                  false, "deprecatedAndHidden": false, "digits": 0, "displayLocationInDecimal":
+                  false, "encrypted": false, "externalId": false, "extraTypeInfo": null, "filterable":
+                  true, "filteredLookupInfo": null, "formulaTreatNullNumberAsZero": false, "groupable":
+                  true, "highScaleNumber": false, "htmlFormatted": false, "idLookup": true,
+                  "inlineHelpText": null, "label": "Account ID", "length": 18, "mask": null,
+                  "maskType": null, "name": "Id", "nameField": false, "namePointing": false,
+                  "nillable": false, "permissionable": false, "picklistValues": [], "polymorphicForeignKey":
+                  false, "precision": 0, "queryByDistance": false, "referenceTargetField": null,
+                  "referenceTo": [], "relationshipName": null, "relationshipOrder": null, "restrictedDelete":
+                  false, "restrictedPicklist": false, "scale": 0, "searchPrefilterable": false,
+                  "soapType": "tns:ID", "sortable": true, "type": "id", "unique": false, "updateable":
+                  false, "writeRequiresMasterRead": false}], "hasSubtypes": false, "implementedBy":
+                  null, "implementsInterfaces": null, "isInterface": false, "isSubtype": false,
+                  "keyPrefix": "001", "label": "Account", "labelPlural": "Accounts", "layoutable":
+                  true, "listviewable": null, "lookupLayoutable": null, "mergeable": true, "mruEnabled":
+                  true, "name": "Account", "namedLayoutInfos": [], "networkScopeFieldName":
+                  null, "queryable": true, "replicateable": true, "retrieveable": true, "searchLayoutable":
+                  true, "searchable": true, "sobjectDescribeOption": "FULL", "triggerable":
+                  true, "undeletable": true, "updateable": true}, "httpHeaders": {"ETag": "\"baf7f695\"",
+                  "Last-Modified": "Fri, 14 Aug 2020 20:53:02 GMT"}, "httpStatusCode": 200,
+                  "referenceId": "refAccount"}, {"body": {"actionOverrides": [], "activateable":
+                  false, "compactLayoutable": true, "createable": true, "custom": false, "customSetting":
+                  false, "deepCloneable": false, "defaultImplementation": null, "deletable":
+                  true, "deprecatedAndHidden": false, "extendedBy": null, "extendsInterfaces":
+                  null, "feedEnabled": true, "fields": [{"aggregatable": true, "aiPredictionField":
+                  false, "autoNumber": false, "byteLength": 18, "calculated": false, "calculatedFormula":
+                  null, "cascadeDelete": false, "caseSensitive": false, "compoundFieldName":
+                  null, "controllerName": null, "createable": false, "custom": false, "defaultValue":
+                  null, "defaultValueFormula": null, "defaultedOnCreate": true, "dependentPicklist":
+                  false, "deprecatedAndHidden": false, "digits": 0, "displayLocationInDecimal":
+                  false, "encrypted": false, "externalId": false, "extraTypeInfo": null, "filterable":
+                  true, "filteredLookupInfo": null, "formulaTreatNullNumberAsZero": false, "groupable":
+                  true, "highScaleNumber": false, "htmlFormatted": false, "idLookup": true,
+                  "inlineHelpText": null, "label": "Contact ID", "length": 18, "mask": null,
+                  "maskType": null, "name": "Id", "nameField": false, "namePointing": false,
+                  "nillable": false, "permissionable": false, "picklistValues": [], "polymorphicForeignKey":
+                  false, "precision": 0, "queryByDistance": false, "referenceTargetField": null,
+                  "referenceTo": [], "relationshipName": null, "relationshipOrder": null, "restrictedDelete":
+                  false, "restrictedPicklist": false, "scale": 0, "searchPrefilterable": false,
+                  "soapType": "tns:ID", "sortable": true, "type": "id", "unique": false, "updateable":
+                  false, "writeRequiresMasterRead": false}], "hasSubtypes": false, "implementedBy":
+                  null, "implementsInterfaces": null, "isInterface": false, "isSubtype": false,
+                  "keyPrefix": "003", "label": "Contact", "labelPlural": "Contacts", "layoutable":
+                  true, "listviewable": null, "lookupLayoutable": null, "mergeable": true, "mruEnabled":
+                  true, "name": "Contact", "namedLayoutInfos": [], "networkScopeFieldName":
+                  null, "queryable": true, "replicateable": true, "retrieveable": true, "searchLayoutable":
+                  true, "searchable": true, "sobjectDescribeOption": "FULL", "triggerable":
+                  true, "undeletable": true, "updateable": true}, "httpHeaders": {"ETag": "\"baf7f695\"",
+                  "Last-Modified": "Fri, 14 Aug 2020 20:53:02 GMT"}, "httpStatusCode": 200,
+                  "referenceId": "refContact"}, {"body": {"urls": {"a":"b"}, "actionOverrides": [], "activateable":
+                  false, "compactLayoutable": true, "createable": true, "custom": false, "customSetting":
+                  false, "deepCloneable": false, "defaultImplementation": null, "deletable":
+                  true, "deprecatedAndHidden": false, "extendedBy": null, "extendsInterfaces":
+                  null, "feedEnabled": true, "fields": [{"aggregatable": true, "aiPredictionField":
+                  false, "autoNumber": false, "byteLength": 18, "calculated": false, "calculatedFormula":
+                  null, "cascadeDelete": false, "caseSensitive": false, "compoundFieldName":
+                  null, "controllerName": null, "createable": false, "custom": false, "defaultValue":
+                  null, "defaultValueFormula": null, "defaultedOnCreate": true, "dependentPicklist":
+                  false, "deprecatedAndHidden": false, "digits": 0, "displayLocationInDecimal":
+                  false, "encrypted": false, "externalId": false, "extraTypeInfo": null, "filterable":
+                  true, "filteredLookupInfo": null, "formulaTreatNullNumberAsZero": false, "groupable":
+                  true, "highScaleNumber": false, "htmlFormatted": false, "idLookup": true,
+                  "inlineHelpText": null, "label": "PermissionSet ID", "length": 18, "mask": null,
+                  "maskType": null, "name": "Id", "nameField": false, "namePointing": false,
+                  "nillable": false, "permissionable": false, "picklistValues": [], "polymorphicForeignKey":
+                  false, "precision": 0, "queryByDistance": false, "referenceTargetField": null,
+                  "referenceTo": [], "relationshipName": null, "relationshipOrder": null, "restrictedDelete":
+                  false, "restrictedPicklist": false, "scale": 0, "searchPrefilterable": false,
+                  "soapType": "tns:ID", "sortable": true, "type": "id", "unique": false, "updateable":
+                  false, "writeRequiresMasterRead": false}], "hasSubtypes": false, "implementedBy":
+                  null, "implementsInterfaces": null, "isInterface": false, "isSubtype": false,
+                  "keyPrefix": "001", "label": "PermissionSet", "labelPlural": "PermissionSets", "layoutable":
+                  true, "listviewable": null, "lookupLayoutable": null, "mergeable": true, "mruEnabled":
+                  true, "name": "PermissionSet", "namedLayoutInfos": [], "networkScopeFieldName":
+                  null, "queryable": true, "replicateable": true, "retrieveable": true, "searchLayoutable":
+                  true, "searchable": true, "sobjectDescribeOption": "FULL", "triggerable":
+                  true, "undeletable": true, "updateable": true}, "httpHeaders": {"ETag": "\"baf7f695\"",
+                  "Last-Modified": "Fri, 14 Aug 2020 20:53:02 GMT"}, "httpStatusCode": 200,
+                  "referenceId": "refPermissionSet"}]}'
+          headers:
+              Response-Headers: Elided
+          status:
+              code: 200
+              message: OK
+    - request:
+          body:
+              '{"compositeRequest": [{"method": "GET", "url": "/services/data/v48.0/sobjects/Opportunity/describe",
+              "referenceId": "refOpportunity", "httpHeaders": {"If-Modified-Since": "Sat,
+              1 Jan 2000 00:00:01 GMT"}}, {"method": "GET", "url": "/services/data/v48.0/sobjects/Campaign/describe",
+              "referenceId": "refCampaign", "httpHeaders": {"If-Modified-Since": "Sat, 1 Jan
+              2000 00:00:01 GMT"}}]}'
+          headers:
+              Request-Headers:
+                  - Elided
+          method: POST
+          uri: https://orgname.my.salesforce.com/services/data/v49.0/composite
+      response:
+          body:
+              string:
+                  '{"compositeResponse": [{"body": {"actionOverrides": [], "activateable":
+                  false, "compactLayoutable": true, "createable": true, "custom": false, "customSetting":
+                  false, "deepCloneable": true, "defaultImplementation": null, "deletable":
+                  true, "deprecatedAndHidden": false, "extendedBy": null, "extendsInterfaces":
+                  null, "feedEnabled": true, "fields": [{"aggregatable": true, "aiPredictionField":
+                  false, "autoNumber": false, "byteLength": 18, "calculated": false, "calculatedFormula":
+                  null, "cascadeDelete": false, "caseSensitive": false, "compoundFieldName":
+                  null, "controllerName": null, "createable": false, "custom": false, "defaultValue":
+                  null, "defaultValueFormula": null, "defaultedOnCreate": true, "dependentPicklist":
+                  false, "deprecatedAndHidden": false, "digits": 0, "displayLocationInDecimal":
+                  false, "encrypted": false, "externalId": false, "extraTypeInfo": null, "filterable":
+                  true, "filteredLookupInfo": null, "formulaTreatNullNumberAsZero": false, "groupable":
+                  true, "highScaleNumber": false, "htmlFormatted": false, "idLookup": true,
+                  "inlineHelpText": null, "label": "Opportunity ID", "length": 18, "mask": null,
+                  "maskType": null, "name": "Id", "nameField": false, "namePointing": false,
+                  "nillable": false, "permissionable": false, "picklistValues": [], "polymorphicForeignKey":
+                  false, "precision": 0, "queryByDistance": false, "referenceTargetField": null,
+                  "referenceTo": [], "relationshipName": null, "relationshipOrder": null, "restrictedDelete":
+                  false, "restrictedPicklist": false, "scale": 0, "searchPrefilterable": false,
+                  "soapType": "tns:ID", "sortable": true, "type": "id", "unique": false, "updateable":
+                  false, "writeRequiresMasterRead": false}], "hasSubtypes": false, "implementedBy":
+                  null, "implementsInterfaces": null, "isInterface": false, "isSubtype": false,
+                  "keyPrefix": "006", "label": "Opportunity", "labelPlural": "Opportunities",
+                  "layoutable": true, "listviewable": null, "lookupLayoutable": null, "mergeable":
+                  false, "mruEnabled": true, "name": "Opportunity", "namedLayoutInfos": [],
+                  "networkScopeFieldName": null, "queryable": true, "replicateable": true, "retrieveable":
+                  true, "searchLayoutable": true, "searchable": true, "sobjectDescribeOption":
+                  "FULL", "triggerable": true, "undeletable": true, "updateable": true}, "httpHeaders":
+                  {"ETag": "\"baf7f695\"", "Last-Modified": "Fri, 14 Aug 2020 20:53:02 GMT"},
+                  "httpStatusCode": 200, "referenceId": "refOpportunity"}, {"body": {"actionOverrides":
+                  [], "activateable": false, "compactLayoutable": true, "createable": true,
+                  "custom": false, "customSetting": false, "deepCloneable": true, "defaultImplementation":
+                  null, "deletable": true, "deprecatedAndHidden": false, "extendedBy": null,
+                  "extendsInterfaces": null, "feedEnabled": false, "fields": [{"aggregatable":
+                  true, "aiPredictionField": false, "autoNumber": false, "byteLength": 18, "calculated":
+                  false, "calculatedFormula": null, "cascadeDelete": false, "caseSensitive":
+                  false, "compoundFieldName": null, "controllerName": null, "createable": false,
+                  "custom": false, "defaultValue": null, "defaultValueFormula": null, "defaultedOnCreate":
+                  true, "dependentPicklist": false, "deprecatedAndHidden": false, "digits":
+                  0, "displayLocationInDecimal": false, "encrypted": false, "externalId": false,
+                  "extraTypeInfo": null, "filterable": true, "filteredLookupInfo": null, "formulaTreatNullNumberAsZero":
+                  false, "groupable": true, "highScaleNumber": false, "htmlFormatted": false,
+                  "idLookup": true, "inlineHelpText": null, "label": "Campaign ID", "length":
+                  18, "mask": null, "maskType": null, "name": "Id", "nameField": false, "namePointing":
+                  false, "nillable": false, "permissionable": false, "picklistValues": [], "polymorphicForeignKey":
+                  false, "precision": 0, "queryByDistance": false, "referenceTargetField": null,
+                  "referenceTo": [], "relationshipName": null, "relationshipOrder": null, "restrictedDelete":
+                  false, "restrictedPicklist": false, "scale": 0, "searchPrefilterable": false,
+                  "soapType": "tns:ID", "sortable": true, "type": "id", "unique": false, "updateable":
+                  false, "writeRequiresMasterRead": false}], "hasSubtypes": false, "implementedBy":
+                  null, "implementsInterfaces": null, "isInterface": false, "isSubtype": false,
+                  "keyPrefix": "701", "label": "Campaign", "labelPlural": "Campaigns", "layoutable":
+                  true, "listviewable": null, "lookupLayoutable": null, "mergeable": false,
+                  "mruEnabled": true, "name": "Campaign", "namedLayoutInfos": [], "networkScopeFieldName":
+                  null, "queryable": true, "replicateable": true, "retrieveable": true, "searchLayoutable":
+                  true, "searchable": true, "sobjectDescribeOption": "FULL", "triggerable":
+                  true, "undeletable": true, "updateable": true}, "httpHeaders": {"ETag": "\"baf7f695\"",
+                  "Last-Modified": "Fri, 14 Aug 2020 20:53:02 GMT"}, "httpStatusCode": 200,
+                  "referenceId": "refCampaign"}]}'
+          headers:
+              Response-Headers: Elided
+          status:
+              code: 200
+              message: OK
 version: 1
diff --git a/cumulusci/utils/tests/test_org_schema.py b/cumulusci/utils/tests/test_org_schema.py
index 8303d8cc4..8133da0fb 100644
--- a/cumulusci/utils/tests/test_org_schema.py
+++ b/cumulusci/utils/tests/test_org_schema.py
@@ -12,11 +12,16 @@ from sqlalchemy import create_engine
 
 from cumulusci.salesforce_api.org_schema import (
     BufferedSession,
+    Filters,
     get_org_schema,
     zip_database,
 )
-from cumulusci.salesforce_api.org_schema_models import Base
+from cumulusci.salesforce_api.org_schema_models import Base, SObject
+from cumulusci.tasks.bulkdata.tests.integration_test_utils import ensure_accounts
 from cumulusci.tests.util import FakeUnreliableRequestHandler
+from cumulusci.utils.http.multi_request import HTTPRequestError
+
+ensure_accounts = ensure_accounts  # fixes 4 lint errors at once. Don't hate the player, hate the game.
 
 
 class FakeSF:
@@ -28,12 +33,20 @@ class FakeSF:
     headers = {}
 
     def describe(self):
+        defaults = {"createable": False, "deletable": True, "deepCloneable": True}
+
+        def fake_obj_desc(name, **props):
+            return {**defaults, **props, "name": name}
+
         return {
             "encoding": "UTF-8",
             "maxBatchSize": 200,
             "sobjects": [
-                {"createable": False, "deletable": True, "name": "Account"},
-                {"createable": False, "deletable": True, "name": "Contact"},
+                fake_obj_desc("Account"),
+                fake_obj_desc("Contact"),
+                fake_obj_desc("PermissionSet", deepCloneable=False),
+                fake_obj_desc("Campaign"),
+                fake_obj_desc("Case"),
             ],
         }
 
@@ -56,7 +69,7 @@ def makeFakeCompositeParallelSalesforce(responses):
 
 
 def uncached_responses(responses):
-    """Pretend to load uncached responses from a VCR cassette"""
+    """Pretend to load uncached responses. Use a VCR cassette instead"""
 
     def parse_composite_response(interaction: dict):
         response_body = interaction["response"]["body"]["string"]
@@ -89,7 +102,7 @@ def mock_return_cached_responses():
     )
 
 
-class TestDescribeOrg:
+class TestOrgSchema:
     def setup_class(self):
         cassette = (
             Path(__file__).parent / "cassettes/ManualEdit_test_describe_to_sql.yaml"
@@ -98,7 +111,7 @@ class TestDescribeOrg:
             self.cassette_data = yaml.safe_load(f)
 
     def validate_schema_data(self, schema):
-        assert len(list(schema.sobjects)) == 4
+        assert len(list(schema.sobjects)) == 4, [obj.name for obj in schema.sobjects]
         assert schema["Account"].createable is True
         assert schema["Account"].fields["Id"].aggregatable is True
         assert schema["Account"].labelPlural == "Accounts"
@@ -148,14 +161,16 @@ class TestDescribeOrg:
         with mock_return_uncached_responses(self.cassette_data):
             with get_org_schema(FakeSF(), org_config) as schema:
                 schema.session.execute("insert into sobjects (name) values ('Foo')")
-                assert "Foo" in schema
+                assert "Foo" in [obj.name for obj in schema.session.query(SObject.name)]
                 schema.session._real_commit__()
                 dbpath = schema.engine.url.translate_connect_args()["database"]
                 zip_database(Path(dbpath), schema.path)
             with get_org_schema(FakeSF(), org_config) as schema:
-                assert "Foo" in schema
+                assert "Foo" in [obj.name for obj in schema.session.query(SObject.name)]
             with get_org_schema(FakeSF(), org_config, force_recache=True) as schema:
-                assert "Foo" not in schema
+                assert "Foo" not in [
+                    obj.name for obj in schema.session.query(SObject.name)
+                ]
 
     def test_dict_like(self, org_config):
         with mock_return_uncached_responses(self.cassette_data):
@@ -295,13 +310,126 @@ class TestDescribeOrg:
             "ManualEditTestDescribeOrg.test_minimal_schema.yaml",
             record_mode="none",
         ), get_org_schema(
-            sf, org_config, include=["Account", "Opportunity"], force_recache=True
+            sf,
+            org_config,
+            included_objects=["Account", "Opportunity"],
+            force_recache=True,
         ) as schema:
             assert list(schema.keys()) == ["Account", "Opportunity"]
 
+    def test_filter_by_name(self, sf, org_config):
+        with mock_return_uncached_responses(self.cassette_data):
+            with get_org_schema(
+                FakeSF(),
+                org_config,
+            ) as schema:
+                assert "Account" in schema
+                assert "PermissionSet" in schema
+            with get_org_schema(
+                FakeSF(), org_config, patterns_to_ignore=["%accou%"]
+            ) as schema:
+                assert "Account" not in schema
+                assert "PermissionSet" in schema
+
+    def test_reuse_query(self, sf, org_config):
+        with mock_return_uncached_responses(self.cassette_data):
+            with get_org_schema(
+                FakeSF(), org_config, filters=[Filters.extractable]
+            ) as schema:
+                assert len(tuple(schema.sobjects)) == len(tuple(schema.sobjects))
+
+    def test_filter_not_extractable_implicit(self, sf, org_config):
+        """Permission Sets are an example of an object considered "not extractable" """
+        with mock_return_uncached_responses(self.cassette_data):
+            with get_org_schema(
+                FakeSF(), org_config, filters=[Filters.extractable]
+            ) as schema:
+                assert "Account" in schema
+                assert "PermissionSet" not in schema
+
+    def test_filter_by_arbitrary_property(self, sf, org_config):
+        """Permission Sets are an example of an object considered "not extractable" """
+        with mock_return_uncached_responses(self.cassette_data):
+            with get_org_schema(
+                FakeSF(), org_config, filters=[Filters.deepCloneable]
+            ) as schema:
+                assert "Account" in schema
+                assert "PermissionSet" not in schema
+
+    def test_cached_schema_can_be_filtered(self, sf, org_config):
+        """Permission Sets are an example of an object considered "not extractable" """
+        with mock_return_uncached_responses(self.cassette_data):
+            with get_org_schema(FakeSF(), org_config) as schema:
+                assert "Account" in schema
+                assert "PermissionSet" in schema
+
+            with get_org_schema(
+                FakeSF(), org_config, filters=[Filters.deepCloneable]
+            ) as schema:
+                assert schema.from_cache
+                assert "Account" in schema
+                assert "PermissionSet" not in schema
+                # it should be still in there but hidden
+                assert schema.session.query(SObject).filter(
+                    SObject.name == "PermissionSet"
+                )
+
+            with get_org_schema(FakeSF(), org_config) as schema:
+                assert schema.from_cache
+                assert "Account" in schema
+                assert "PermissionSet" in schema
+
+    def test_error_populate_without_include_counts(self, sf, org_config):
+        with mock_return_uncached_responses(self.cassette_data):
+            with pytest.raises(AssertionError, match="include_counts"):
+                with get_org_schema(
+                    FakeSF(), org_config, filters=[Filters.populated]
+                ) as schema:
+                    assert "Account" in schema
+                    assert "PermissionSet" not in schema
+
+    def test_filter_by_populated(self, sf, org_config):
+        with mock_return_uncached_responses(self.cassette_data):
+            with patch(
+                "cumulusci.salesforce_api.org_schema.count_sobjects",
+                lambda *args: (
+                    {"Account": 10, "Contact": 5, "PermissionSet": 0},
+                    [],
+                    [],
+                ),
+            ), get_org_schema(
+                FakeSF(), org_config, include_counts=True, filters=[Filters.populated]
+            ) as schema:
+                assert "Account" in schema
+                assert "PermissionSet" not in schema
+
+    def test_error_while_counting(self, sf, org_config, caplog):
+        with mock_return_uncached_responses(self.cassette_data):
+            with patch(
+                "cumulusci.salesforce_api.org_schema.count_sobjects",
+                lambda *args: (
+                    {"Account": 10, "Contact": 5, "PermissionSet": 0},
+                    [],
+                    [HTTPRequestError("Error! Apostasy!", None)] * 15,
+                ),
+            ), get_org_schema(
+                FakeSF(), org_config, include_counts=True, filters=[Filters.populated]
+            ):
+                pass
+            assert "Apostasy" in caplog.text
+            assert "more counting errors suppressed" in caplog.text
+
+    @pytest.mark.needs_org()
+    def test_schema_populated_real(self, sf, org_config, ensure_accounts):
+        with ensure_accounts(6):
+            with get_org_schema(
+                sf, org_config, include_counts=True, filters=[Filters.populated]
+            ) as schema:
+                assert "Account" in schema
+                assert "Case" not in schema
+
 
-@pytest.mark.vcr()  # too hard to make these VCR-compatible due to data volume
-@pytest.mark.slow()
+@pytest.mark.needs_org()  # too hard to make these VCR-compatible due to data volume
 class TestOrgSchemaIntegration:
     def validate_real_schema_data(self, schema):
         assert len(list(schema.sobjects)) > 800
@@ -323,7 +451,10 @@ class TestOrgSchemaIntegration:
 
     def test_minimal_schema(self, sf, org_config):
         with get_org_schema(
-            sf, org_config, include=["Account", "Opportunity"], force_recache=True
+            sf,
+            org_config,
+            included_objects=["Account", "Opportunity"],
+            force_recache=True,
         ) as schema:
             assert list(schema.keys()) == ["Account", "Opportunity"]