Repository: pulp/pulpcore

Issue:
Cache Content qs results during ContentArtifactResource import
**Version**
Please provide the versions of the pulpcore and plugin packages in use, and how they are installed.  If you are using Pulp via Katello, please provide the Katello version.

**Describe the bug**
A clear and concise description of what the bug is.

**To Reproduce**
Steps to reproduce the behavior:

**Expected behavior**
A clear and concise description of what you expected to happen.

**Additional context**
Add any other context about the problem here.  Please provide links to any previous discussions via Discourse or Bugzilla.



Discussion:

Resolving patch:
diff --git a/pulpcore/app/modelresource.py b/pulpcore/app/modelresource.py
--- a/pulpcore/app/modelresource.py
+++ b/pulpcore/app/modelresource.py
@@ -86,9 +86,12 @@ class ContentArtifactResource(QueryModelResource):
     artifact = fields.Field(
         column_name="artifact", attribute="artifact", widget=ForeignKeyWidget(Artifact, "sha256")
     )
+    linked_content = {}
 
     def __init__(self, repo_version=None, content_mapping=None):
         self.content_mapping = content_mapping
+        if not ContentArtifactResource.linked_content:
+            ContentArtifactResource.linked_content = self.fetch_linked_content()
         super().__init__(repo_version)
 
     def before_import_row(self, row, **kwargs):
@@ -108,8 +111,7 @@ def before_import_row(self, row, **kwargs):
         """
         super().before_import_row(row, **kwargs)
 
-        linked_content = Content.objects.get(upstream_id=row["content"])
-        row["content"] = str(linked_content.pulp_id)
+        row["content"] = self.linked_content[row["content"]]
 
     def set_up_queryset(self):
         vers_content = ContentArtifact.objects.filter(content__in=self.repo_version.content)
@@ -122,6 +124,14 @@ def set_up_queryset(self):
             )
         return vers_content.order_by("content", "relative_path")
 
+    def fetch_linked_content(self):
+        linked_content = {}
+        c_qs = Content.objects.filter(upstream_id__isnull=False).values("upstream_id", "pulp_id")
+        for c in c_qs.iterator():
+            linked_content[str(c["upstream_id"])] = str(c["pulp_id"])
+
+        return linked_content
+
     class Meta:
         model = ContentArtifact
         import_id_fields = (
diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py
--- a/pulpcore/app/tasks/importer.py
+++ b/pulpcore/app/tasks/importer.py
@@ -225,6 +225,16 @@ def import_repository_version(
             raise ValidationError(_("No RepositoryVersion found for {}").format(rv_name))
 
         rv_path = os.path.join(temp_dir, rv_name)
+
+        # see if we have a Content mapping
+        mapping_path = f"{rv_name}/{CONTENT_MAPPING_FILE}"
+        mapping = {}
+        with tarfile.open(tar_path, "r") as tar:
+            if mapping_path in tar.getnames():
+                tar.extract(mapping_path, path=temp_dir)
+                with open(os.path.join(temp_dir, mapping_path), "r") as mapping_file:
+                    mapping = json.load(mapping_file)
+
         # Content
         plugin_name = src_repo_type.split(".")[0]
         cfg = get_plugin_config(plugin_name)
@@ -241,7 +251,7 @@ def import_repository_version(
                     if repo_resource.import_type in ("new", "update"):
                         dest_repo_pk = repo_resource.object_id
 
-                if issubclass(res_class, BaseContentResource):
+                if not mapping and issubclass(res_class, BaseContentResource):
                     resulting_content_ids.extend(
                         row.object_id
                         for row in a_result.rows
@@ -254,15 +264,6 @@ def import_repository_version(
         for a_batch in _import_file(ca_path, ContentArtifactResource, retry=True):
             pass
 
-        # see if we have a content mapping
-        mapping_path = f"{rv_name}/{CONTENT_MAPPING_FILE}"
-        mapping = {}
-        with tarfile.open(tar_path, "r") as tar:
-            if mapping_path in tar.getnames():
-                tar.extract(mapping_path, path=temp_dir)
-                with open(os.path.join(temp_dir, mapping_path), "r") as mapping_file:
-                    mapping = json.load(mapping_file)
-
         content_count = 0
         if mapping:
             # use the content mapping to map content to repos
@@ -270,14 +271,14 @@ def import_repository_version(
                 repo_name = _get_destination_repo_name(importer, repo_name)
                 dest_repo = Repository.objects.get(name=repo_name)
                 content = Content.objects.filter(upstream_id__in=content_ids)
-                content_count += content.count()
+                content_count += len(content_ids)
                 with dest_repo.new_version() as new_version:
                     new_version.set_content(content)
         else:
             # just map all the content to our destination repo
             dest_repo = Repository.objects.get(pk=dest_repo_pk)
             content = Content.objects.filter(pk__in=resulting_content_ids)
-            content_count += content.count()
+            content_count += len(resulting_content_ids)
             with dest_repo.new_version() as new_version:
                 new_version.set_content(content)
 


Test patch: