Repository: pulp/pulpcore

Issue:
Provide start_version  and end version for fs exporter.
**Is your feature request related to a problem? Please describe.**
Pulp today can export a full yum repository with fs-exporter.  You pass it a publication and it will export it. However  customers want the 'incremental export' option for this exporter similar to the way the regular exporter functions. 
Notice how regular pulp exporter takes `versions` and  `start_versions` parameter here => https://docs.pulpproject.org/pulpcore/restapi.html#tag/Exporters:-Pulp-Exports/operation/exporters_core_pulp_exports_create
Pulp exports everything from there. We would need something similar for fs exporters also and will simpler because fs export concentrates on a single repo instead of multiple as the regular one does.

**Describe the solution you'd like**
Check out https://docs.pulpproject.org/pulpcore/restapi.html#tag/Exporters:-Filesystem-Exports/operation/exporters_core_filesystem_exports_create

Make the export take a publication as it does today and a start repository version

Ideally whatever is exported 

1. Will be delta between `publication_version - start_version` in terms of rpms (mostly added)
2.  The metadata of publication_version will be copied verbatim (no comparison necessary)

So this repository export  is not 'complete' since it has only the added/changed rpms but the metadata of the suggested publication.

Idea is 
* User has already exported Rhel - 7 (using the fs exporter) - full export 
* User consumes content from the export
* Several days later the user now wants the new content for RHEL 7. Instead of exporting 50GB worth of content again the user's export should only consist of rpms that got added after the start version and latest metadata 
* User should be able to copy the contents of this export over the regular rhel export directory 
and use that to sync repodata.

**Describe alternatives you've considered**

I thought about other approaches but I think providing a start_version is the least invasive way imo that adequately serves us.


**Additional context**
The popularity of the fs export approach has made adding this functionality  necessary. Recommend giving it a high priority.

https://bugzilla.redhat.com/show_bug.cgi?id=2143497


Discussion:

Resolving patch:
diff --git a/pulpcore/app/serializers/exporter.py b/pulpcore/app/serializers/exporter.py
--- a/pulpcore/app/serializers/exporter.py
+++ b/pulpcore/app/serializers/exporter.py
@@ -294,6 +294,12 @@ class FilesystemExportSerializer(ExportSerializer):
         write_only=True,
     )
 
+    start_repository_version = RepositoryVersionRelatedField(
+        help_text=_("The URI of the last-exported-repo-version."),
+        required=False,
+        write_only=True,
+    )
+
     def validate(self, data):
         if ("publication" not in data and "repository_version" not in data) or (
             "publication" in data and "repository_version" in data
@@ -305,7 +311,11 @@ def validate(self, data):
 
     class Meta:
         model = models.FilesystemExport
-        fields = ExportSerializer.Meta.fields + ("publication", "repository_version")
+        fields = ExportSerializer.Meta.fields + (
+            "publication",
+            "repository_version",
+            "start_repository_version",
+        )
 
 
 class FilesystemExporterSerializer(ExporterSerializer):
diff --git a/pulpcore/app/tasks/export.py b/pulpcore/app/tasks/export.py
--- a/pulpcore/app/tasks/export.py
+++ b/pulpcore/app/tasks/export.py
@@ -105,7 +105,7 @@ def _export_to_file_system(path, relative_paths_to_artifacts, method=FS_EXPORT_M
 
 
 def _export_publication_to_file_system(
-    path, publication, method=FS_EXPORT_METHODS.WRITE, allow_missing=False
+    path, publication, start_repo_version=None, method=FS_EXPORT_METHODS.WRITE, allow_missing=False
 ):
     """
     Export a publication to the file system.
@@ -114,9 +114,19 @@ def _export_publication_to_file_system(
         path (str): Path to place the exported data
         publication_pk (str): Publication pk
     """
+    difference_content_artifacts = []
     content_artifacts = ContentArtifact.objects.filter(
         pk__in=publication.published_artifact.values_list("content_artifact__pk", flat=True)
     )
+    if start_repo_version:
+        start_version_content_artifacts = ContentArtifact.objects.filter(
+            artifact__in=start_repo_version.artifacts
+        )
+        difference_content_artifacts = set(
+            content_artifacts.difference(start_version_content_artifacts).values_list(
+                "pk", flat=True
+            )
+        )
 
     if publication.pass_through:
         content_artifacts |= ContentArtifact.objects.filter(
@@ -138,8 +148,11 @@ def _export_publication_to_file_system(
         "content_artifact", "content_artifact__artifact"
     ).iterator():
         # Artifact isn't guaranteed to be present
-        if pa.content_artifact.artifact:
+        if pa.content_artifact.artifact and (
+            start_repo_version is None or pa.content_artifact.pk in difference_content_artifacts
+        ):
             relative_path_to_artifacts[pa.relative_path] = pa.content_artifact.artifact
+
     _export_to_file_system(path, relative_path_to_artifacts, method)
 
 
@@ -160,7 +173,7 @@ def _export_location_is_clean(path):
     return True
 
 
-def fs_publication_export(exporter_pk, publication_pk):
+def fs_publication_export(exporter_pk, publication_pk, start_repo_version_pk=None):
     """
     Export a publication to the file system using an exporter.
 
@@ -170,25 +183,41 @@ def fs_publication_export(exporter_pk, publication_pk):
     """
     exporter = Exporter.objects.get(pk=exporter_pk).cast()
     publication = Publication.objects.get(pk=publication_pk).cast()
+
+    start_repo_version = None
+    if start_repo_version_pk:
+        start_repo_version = RepositoryVersion.objects.get(pk=start_repo_version_pk)
+
+    params = {"publication": publication_pk}
+    if start_repo_version:
+        params["start_repository_version"] = start_repo_version_pk
+
     export = FilesystemExport.objects.create(
         exporter=exporter,
-        params={"publication": publication_pk},
+        params=params,
         task=Task.current(),
     )
     ExportedResource.objects.create(export=export, content_object=publication)
     CreatedResource.objects.create(content_object=export)
-
     log.info(
-        "Exporting: file_system_exporter={exporter}, publication={publication}, path={path}".format(
-            exporter=exporter.name, publication=publication.pk, path=exporter.path
+        "Exporting: file_system_exporter={exporter}, publication={publication}, "
+        "start_repo_version={start_repo_version}, path={path}".format(
+            exporter=exporter.name,
+            publication=publication.pk,
+            start_repo_version=start_repo_version_pk,
+            path=exporter.path,
         )
     )
+
     if not _export_location_is_clean(exporter.path):
         raise RuntimeError(_("Cannot export to directories that contain existing data."))
-    _export_publication_to_file_system(exporter.path, publication, exporter.method)
+
+    _export_publication_to_file_system(
+        exporter.path, publication, start_repo_version=start_repo_version, method=exporter.method
+    )
 
 
-def fs_repo_version_export(exporter_pk, repo_version_pk):
+def fs_repo_version_export(exporter_pk, repo_version_pk, start_repo_version_pk=None):
     """
     Export a repository version to the file system using an exporter.
 
@@ -198,9 +227,17 @@ def fs_repo_version_export(exporter_pk, repo_version_pk):
     """
     exporter = Exporter.objects.get(pk=exporter_pk).cast()
     repo_version = RepositoryVersion.objects.get(pk=repo_version_pk)
+    start_repo_version = None
+    if start_repo_version_pk:
+        start_repo_version = RepositoryVersion.objects.get(pk=start_repo_version_pk)
+
+    params = {"repository_version": repo_version_pk}
+    if start_repo_version:
+        params["start_repository_version"] = start_repo_version_pk
+
     export = FilesystemExport.objects.create(
         exporter=exporter,
-        params={"repository_version": repo_version_pk},
+        params=params,
         task=Task.current(),
     )
     ExportedResource.objects.create(export=export, content_object=repo_version)
@@ -208,18 +245,31 @@ def fs_repo_version_export(exporter_pk, repo_version_pk):
 
     log.info(
         "Exporting: file_system_exporter={exporter}, repo_version={repo_version}, "
-        "path={path}".format(
-            exporter=exporter.name, repo_version=repo_version.pk, path=exporter.path
+        "start_repo_version={start_repo_version}, path={path}".format(
+            exporter=exporter.name,
+            repo_version=repo_version.pk,
+            start_repo_version=start_repo_version_pk,
+            path=exporter.path,
         )
     )
 
     content_artifacts = ContentArtifact.objects.filter(content__in=repo_version.content)
     _validate_fs_export(content_artifacts)
+    difference_content_artifacts = []
+    if start_repo_version:
+        start_version_content_artifacts = ContentArtifact.objects.filter(
+            artifact__in=start_repo_version.artifacts
+        )
+        difference_content_artifacts = set(
+            content_artifacts.difference(start_version_content_artifacts).values_list(
+                "pk", flat=True
+            )
+        )
 
-    relative_path_to_artifacts = {
-        ca.relative_path: ca.artifact
-        for ca in content_artifacts.select_related("artifact").iterator()
-    }
+    relative_path_to_artifacts = {}
+    for ca in content_artifacts.select_related("artifact").iterator():
+        if start_repo_version is None or ca.pk in difference_content_artifacts:
+            relative_path_to_artifacts[ca.relative_path] = ca.artifact
 
     _export_to_file_system(exporter.path, relative_path_to_artifacts, exporter.method)
 
diff --git a/pulpcore/app/viewsets/exporter.py b/pulpcore/app/viewsets/exporter.py
--- a/pulpcore/app/viewsets/exporter.py
+++ b/pulpcore/app/viewsets/exporter.py
@@ -159,13 +159,22 @@ def create(self, request, exporter_pk):
         serializer = FilesystemExportSerializer(data=request.data, context={"exporter": exporter})
         serializer.is_valid(raise_exception=True)
 
+        start_repository_version_pk = None
+        if request.data.get("start_repository_version"):
+            start_repository_version_pk = self.get_resource(
+                request.data["start_repository_version"], RepositoryVersion
+            ).pk
+
         if request.data.get("publication"):
             publication = self.get_resource(request.data["publication"], Publication)
-
             task = dispatch(
                 fs_publication_export,
                 exclusive_resources=[exporter],
-                kwargs={"exporter_pk": exporter.pk, "publication_pk": publication.pk},
+                kwargs={
+                    "exporter_pk": exporter.pk,
+                    "publication_pk": publication.pk,
+                    "start_repo_version_pk": start_repository_version_pk,
+                },
             )
         else:
             repo_version = self.get_resource(request.data["repository_version"], RepositoryVersion)
@@ -173,7 +182,11 @@ def create(self, request, exporter_pk):
             task = dispatch(
                 fs_repo_version_export,
                 exclusive_resources=[exporter],
-                kwargs={"exporter_pk": str(exporter.pk), "repo_version_pk": repo_version.pk},
+                kwargs={
+                    "exporter_pk": str(exporter.pk),
+                    "repo_version_pk": repo_version.pk,
+                    "start_repo_version_pk": start_repository_version_pk,
+                },
             )
 
         return OperationPostponedResponse(task, request)


Test patch: