internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: pulp/pulpcore

Issue:
An error is not raised when exporting remote artifacts



Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/export.py b/pulpcore/app/tasks/export.py
--- a/pulpcore/app/tasks/export.py
+++ b/pulpcore/app/tasks/export.py
@@ -50,7 +50,7 @@ def _export_to_file_system(path, content_artifacts, method=FS_EXPORT_METHODS.WRI
         RuntimeError: If Artifacts are not downloaded or when trying to link non-fs files
     """
     if content_artifacts.filter(artifact=None).exists():
-        RuntimeError(_("Cannot export artifacts that haven't been downloaded."))
+        raise RuntimeError(_("Cannot export artifacts that haven't been downloaded."))
 
     if (
         settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem"
@@ -369,7 +369,7 @@ def _do_export(pulp_exporter, tar, the_export):
         # an on_demand repo
         content_artifacts = ContentArtifact.objects.filter(content__in=version.content)
         if content_artifacts.filter(artifact=None).exists():
-            RuntimeError(_("Remote artifacts cannot be exported."))
+            raise RuntimeError(_("Remote artifacts cannot be exported."))
 
         if do_incremental:
             vers_artifacts = version.artifacts.difference(vers_match[version].artifacts).all()


Test patch:
Repository: pulp/pulpcore

Issue:
An error is not raised when exporting remote artifacts



Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/export.py b/pulpcore/app/tasks/export.py
--- a/pulpcore/app/tasks/export.py
+++ b/pulpcore/app/tasks/export.py
@@ -296,7 +296,7 @@ def _do_export(pulp_exporter, tar, the_export):
         # an on_demand repo
         content_artifacts = ContentArtifact.objects.filter(content__in=version.content)
         if content_artifacts.filter(artifact=None).exists():
-            RuntimeError(_("Remote artifacts cannot be exported."))
+            raise RuntimeError(_("Remote artifacts cannot be exported."))
 
         if do_incremental:
             vers_artifacts = version.artifacts.difference(vers_match[version].artifacts).all()


Test patch:
Repository: pulp/pulpcore

Issue:
Content app is making queries on head requests
```
(pulp) [vagrant@pulp3-source-centos8-stream-fips ~]$ nc 127.0.0.1 4444                                                                                                                   [8/8]
> /home/vagrant/devel/pulpcore/pulpcore/content/handler.py(144)list_distributions()                                                                                                           
-> return HTTPOk(headers={"Content-Type": "text/html"}, body=self.render_html(directory_list))                                                                                                
(Pdb) directory_list                                                                                                                                                                          
['58016a04-8204-4ced-8f87-5318716e525d/']                                                                                                                                                     
(Pdb) base_paths                                                                                                                                                                              
['58016a04-8204-4ced-8f87-5318716e525d']            
(Pdb) request.method
'HEAD'
(Pdb) c
```


Discussion:

Resolving patch:
diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py
--- a/pulpcore/content/handler.py
+++ b/pulpcore/content/handler.py
@@ -138,6 +138,8 @@ def get_base_paths_blocking():
                 )
             return base_paths
 
+        if request.method.lower() == "head":
+            return HTTPOk(headers={"Content-Type": "text/html"})
         base_paths = await sync_to_async(get_base_paths_blocking)()
         directory_list = ["{}/".format(path) for path in base_paths]
         return HTTPOk(headers={"Content-Type": "text/html"}, body=self.render_html(directory_list))


Test patch:
Repository: pulp/pulpcore

Issue:
Content app is making queries on head requests
```
(pulp) [vagrant@pulp3-source-centos8-stream-fips ~]$ nc 127.0.0.1 4444                                                                                                                   [8/8]
> /home/vagrant/devel/pulpcore/pulpcore/content/handler.py(144)list_distributions()                                                                                                           
-> return HTTPOk(headers={"Content-Type": "text/html"}, body=self.render_html(directory_list))                                                                                                
(Pdb) directory_list                                                                                                                                                                          
['58016a04-8204-4ced-8f87-5318716e525d/']                                                                                                                                                     
(Pdb) base_paths                                                                                                                                                                              
['58016a04-8204-4ced-8f87-5318716e525d']            
(Pdb) request.method
'HEAD'
(Pdb) c
```


Discussion:

Resolving patch:
diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py
--- a/pulpcore/content/handler.py
+++ b/pulpcore/content/handler.py
@@ -132,6 +132,8 @@ def get_base_paths_blocking():
                 )
             return base_paths
 
+        if request.method.lower() == "head":
+            return HTTPOk(headers={"Content-Type": "text/html"})
         base_paths = await sync_to_async(get_base_paths_blocking)()
         directory_list = ["{}/".format(path) for path in base_paths]
         return HTTPOk(headers={"Content-Type": "text/html"}, body=self.render_html(directory_list))


Test patch:
Repository: pulp/pulpcore

Issue:
Content app is making queries on head requests
```
(pulp) [vagrant@pulp3-source-centos8-stream-fips ~]$ nc 127.0.0.1 4444                                                                                                                   [8/8]
> /home/vagrant/devel/pulpcore/pulpcore/content/handler.py(144)list_distributions()                                                                                                           
-> return HTTPOk(headers={"Content-Type": "text/html"}, body=self.render_html(directory_list))                                                                                                
(Pdb) directory_list                                                                                                                                                                          
['58016a04-8204-4ced-8f87-5318716e525d/']                                                                                                                                                     
(Pdb) base_paths                                                                                                                                                                              
['58016a04-8204-4ced-8f87-5318716e525d']            
(Pdb) request.method
'HEAD'
(Pdb) c
```


Discussion:

Resolving patch:
diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py
--- a/pulpcore/content/handler.py
+++ b/pulpcore/content/handler.py
@@ -131,6 +131,8 @@ def get_base_paths_blocking():
                 )
             return base_paths
 
+        if request.method.lower() == "head":
+            return HTTPOk(headers={"Content-Type": "text/html"})
         base_paths = await sync_to_async(get_base_paths_blocking)()
         directory_list = ["{}/".format(path) for path in base_paths]
         return HTTPOk(headers={"Content-Type": "text/html"}, body=self.render_html(directory_list))


Test patch:
Repository: pulp/pulpcore

Issue:
Add another field to the `RepositoryVersion` model to store additional information.
**Is your feature request related to a problem? Please describe.**

We are looking to implement an optimize sync feature in `pulp_deb`. In order to do that we want to store data in a `dict` format which contains information about the remote options used in the latest sync as well as `sha256` values for the repository files. These will be used to compare when trying to sync a repository again to see if something has changed and skip certain steps if it hasn't.

**Describe the solution you'd like**

Adding another field to `RepositoryVersion` model that can store a `.JSONField(default=dict)`. This way a repository can always look up the lastest version to get the previous data.

**Describe alternatives you've considered**

`pulp_rpm` uses their `Repository` model to save this data which works too. However there were concerns of the mutability of the data when stored this way.

Another solution would be adding a subclass of `RepositoryVersion` to `pulp_deb`. Since there is not any subclass of this model present in any plugin so far, this seems a bit overkill to just add one additional field.

**Additional context**

For context the PR for this feature in `pulp_deb`: https://github.com/pulp/pulp_deb/pull/570



Discussion:

Resolving patch:
diff --git a/pulpcore/app/migrations/0093_add_info_field_repositoryversion.py b/pulpcore/app/migrations/0093_add_info_field_repositoryversion.py
new file mode 100644
--- /dev/null
+++ b/pulpcore/app/migrations/0093_add_info_field_repositoryversion.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.13 on 2022-08-02 15:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0092_alter_upload_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='repositoryversion',
+            name='info',
+            field=models.JSONField(default=dict),
+        ),
+    ]
diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py
--- a/pulpcore/app/models/repository.py
+++ b/pulpcore/app/models/repository.py
@@ -605,6 +605,7 @@ class RepositoryVersion(BaseModel):
     number = models.PositiveIntegerField(db_index=True)
     complete = models.BooleanField(db_index=True, default=False)
     base_version = models.ForeignKey("RepositoryVersion", null=True, on_delete=models.SET_NULL)
+    info = models.JSONField(default=dict)
 
     class Meta:
         default_related_name = "versions"


Test patch:
Repository: pulp/pulpcore

Issue:
Import attempts to add everything, not only content, to the target repository



Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py
--- a/pulpcore/app/tasks/importer.py
+++ b/pulpcore/app/tasks/importer.py
@@ -36,6 +36,8 @@
 from pulpcore.constants import TASK_STATES
 from pulpcore.tasking.tasks import dispatch
 
+from pulpcore.plugin.importexport import BaseContentResource
+
 log = getLogger(__name__)
 
 ARTIFACT_FILE = "pulpcore.app.modelresource.ArtifactResource.json"
@@ -236,13 +238,14 @@ def import_repository_version(importer_pk, destination_repo_pk, source_repo_name
 
         resulting_content_ids = []
         for res_class in cfg.exportable_classes:
-            content_count = 0
             filename = f"{res_class.__module__}.{res_class.__name__}.json"
             for a_result in _import_file(os.path.join(rv_path, filename), res_class, retry=True):
-                content_count += len(a_result.rows)
-                resulting_content_ids.extend(
-                    row.object_id for row in a_result.rows if row.import_type in ("new", "update")
-                )
+                if issubclass(res_class, BaseContentResource):
+                    resulting_content_ids.extend(
+                        row.object_id
+                        for row in a_result.rows
+                        if row.import_type in ("new", "update")
+                    )
 
         # Once all content exists, create the ContentArtifact links
         ca_path = os.path.join(rv_path, CA_FILE)


Test patch:
Repository: pulp/pulpcore

Issue:
django-admin dump-permissions command fails
`
django-admin dump-permissions  --settings=pulpcore.app.settings
Traceback (most recent call last):
  File "/var/lib/pulp/pulpvenv/bin/django-admin", line 8, in <module>
    sys.exit(execute_from_command_line())
  File "/var/lib/pulp/pulpvenv/lib64/python3.8/site-packages/django/core/management/__init__.py", line 419, in execute_from_command_line
    utility.execute()
  File "/var/lib/pulp/pulpvenv/lib64/python3.8/site-packages/django/core/management/__init__.py", line 413, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/var/lib/pulp/pulpvenv/lib64/python3.8/site-packages/django/core/management/base.py", line 354, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/var/lib/pulp/pulpvenv/lib64/python3.8/site-packages/django/core/management/base.py", line 398, in execute
    output = self.handle(*args, **options)
  File "/var/lib/pulp/pulpvenv/lib64/python3.8/site-packages/pulpcore/app/management/commands/dump-permissions.py", line 231, in handle
    data["group_object_permissions"] = list(_get_group_object_permissions())
  File "/var/lib/pulp/pulpvenv/lib64/python3.8/site-packages/pulpcore/app/management/commands/dump-permissions.py", line 156, in _get_group_object_permissions
    yield {"groupname": groupname, "object": obj, "permissions": permissions}
UnboundLocalError: local variable 'groupname' referenced before assignment
`


Discussion:

Resolving patch:
diff --git a/pulpcore/app/management/commands/dump-permissions.py b/pulpcore/app/management/commands/dump-permissions.py
--- a/pulpcore/app/management/commands/dump-permissions.py
+++ b/pulpcore/app/management/commands/dump-permissions.py
@@ -153,7 +153,7 @@ def _get_group_object_permissions():
                 obj = _get_url(content_type_id, object_pk)
             except Exception:
                 obj = f"{app_label}.{model}:{object_pk}"
-        yield {"groupname": groupname, "object": obj, "permissions": permissions}
+            yield {"groupname": groupname, "object": obj, "permissions": permissions}
 
 
 class Command(BaseCommand):


Test patch:
Repository: pulp/pulpcore

Issue:
general_create performes one unnecessary db lookup
`Serializer.save()` returns the instance we want already. No need to fetch it again from the db.


Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/base.py b/pulpcore/app/tasks/base.py
--- a/pulpcore/app/tasks/base.py
+++ b/pulpcore/app/tasks/base.py
@@ -35,8 +35,7 @@ def general_create(app_label, serializer_name, *args, **kwargs):
     serializer_class = get_plugin_config(app_label).named_serializers[serializer_name]
     serializer = serializer_class(data=data, context=context)
     serializer.is_valid(raise_exception=True)
-    serializer.save()
-    instance = serializer_class.Meta.model.objects.get(pk=serializer.instance.pk).cast()
+    instance = serializer.save().cast()
     resource = CreatedResource(content_object=instance)
     resource.save()
 


Test patch:
Repository: pulp/pulpcore

Issue:
The same content is imported multiple times in a row
Exported resources are imported `MAX_ATTEMPTS` times in a row which is unnecessary:

https://github.com/pulp/pulpcore/blob/main/pulpcore/app/tasks/importer.py#L107-L133

Introducing a new `yield`ing else branch would help to resolve this problem.


Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py
--- a/pulpcore/app/tasks/importer.py
+++ b/pulpcore/app/tasks/importer.py
@@ -123,14 +123,17 @@ def _import_file(fpath, resource_class, retry=False):
                                     curr_attempt=curr_attempt,
                                 )
                             )
-
-                    # Last attempt, we raise an exception on any problem.
-                    # This will either succeed, or log a fatal error and fail.
-                    try:
-                        a_result = resource.import_data(data, raise_errors=True)
-                    except Exception as e:  # noqa log on ANY exception and then re-raise
-                        log.error(f"FATAL import-failure importing {fpath}")
-                        raise
+                        else:
+                            break
+                    else:
+                        # The while condition is not fulfilled, so we proceed to the last attempt,
+                        # we raise an exception on any problem. This will either succeed, or log a
+                        # fatal error and fail.
+                        try:
+                            a_result = resource.import_data(data, raise_errors=True)
+                        except Exception as e:  # noqa log on ANY exception and then re-raise
+                            log.error(f"FATAL import-failure importing {fpath}")
+                            raise
                 else:
                     a_result = resource.import_data(data, raise_errors=True)
                 yield a_result


Test patch:
Repository: pulp/pulpcore

Issue:
Reuse existing artifact and content units in case the user provided proof that they have the bits
On content upload, pulp should not reject the request for any uniqueness constraint violation, but realize that the user was able to provide the bits and therefor grant using the existing records, which means not throwing an exception and returning the href of the existing object, possibly also adding it into a repository if it was specified.

This is a generalization of https://github.com/pulp/pulp_file/issues/774



Discussion:

Resolving patch:
diff --git a/pulpcore/app/serializers/content.py b/pulpcore/app/serializers/content.py
--- a/pulpcore/app/serializers/content.py
+++ b/pulpcore/app/serializers/content.py
@@ -47,22 +47,37 @@ def __init__(self, *args, **kwargs):
     @transaction.atomic
     def create(self, validated_data):
         """
-        Create the content and associate it with its Artifact.
+        Create the content and associate it with its Artifact, or retrieve the existing content.
 
         Args:
             validated_data (dict): Data to save to the database
         """
-        artifact = validated_data.pop("artifact")
-        if "relative_path" not in self.fields or self.fields["relative_path"].write_only:
-            relative_path = validated_data.pop("relative_path")
+        content = self.retrieve(validated_data)
+
+        if content is not None:
+            content.touch()
         else:
-            relative_path = validated_data.get("relative_path")
-        content = self.Meta.model.objects.create(**validated_data)
-        models.ContentArtifact.objects.create(
-            artifact=artifact, content=content, relative_path=relative_path
-        )
+            artifact = validated_data.pop("artifact")
+            if "relative_path" not in self.fields or self.fields["relative_path"].write_only:
+                relative_path = validated_data.pop("relative_path")
+            else:
+                relative_path = validated_data.get("relative_path")
+            content = self.Meta.model.objects.create(**validated_data)
+            models.ContentArtifact.objects.create(
+                artifact=artifact, content=content, relative_path=relative_path
+            )
+
         return content
 
+    def retrieve(self, validated_data):
+        """
+        Retrieve existing content unit if it exists, else return None.
+
+        This method is plugin-specific and implementing it for a specific content type
+        allows for uploading already existing content units of that type.
+        """
+        return None
+
     class Meta:
         model = models.Content
         fields = BaseContentSerializer.Meta.fields + ("artifact", "relative_path")


Test patch:
Repository: pulp/pulpcore

Issue:
Add database layer protection to RepositoryContent



Discussion:

Resolving patch:
diff --git a/pulpcore/app/migrations/0094_protect_repository_content.py b/pulpcore/app/migrations/0094_protect_repository_content.py
new file mode 100644
--- /dev/null
+++ b/pulpcore/app/migrations/0094_protect_repository_content.py
@@ -0,0 +1,29 @@
+# Generated by Django 3.2.15 on 2022-08-18 08:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0093_add_info_field_repositoryversion'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='repositorycontent',
+            name='content',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='version_memberships', to='core.content'),
+        ),
+        migrations.AlterField(
+            model_name='repositorycontent',
+            name='version_added',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, related_name='added_memberships', to='core.repositoryversion'),
+        ),
+        migrations.AlterField(
+            model_name='repositorycontent',
+            name='version_removed',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.RESTRICT, related_name='removed_memberships', to='core.repositoryversion'),
+        ),
+    ]
diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py
--- a/pulpcore/app/models/repository.py
+++ b/pulpcore/app/models/repository.py
@@ -521,15 +521,21 @@ class RepositoryContent(BaseModel):
             Content.
     """
 
+    # Content can only be removed once it's no longer referenced by any repository
     content = models.ForeignKey(
-        "Content", on_delete=models.CASCADE, related_name="version_memberships"
+        "Content", on_delete=models.PROTECT, related_name="version_memberships"
     )
     repository = models.ForeignKey(Repository, on_delete=models.CASCADE)
+    # version_added and version_removed need to be properly handled in _squash before the version
+    # can be deleted
     version_added = models.ForeignKey(
-        "RepositoryVersion", related_name="added_memberships", on_delete=models.CASCADE
+        "RepositoryVersion", related_name="added_memberships", on_delete=models.RESTRICT
     )
     version_removed = models.ForeignKey(
-        "RepositoryVersion", null=True, related_name="removed_memberships", on_delete=models.CASCADE
+        "RepositoryVersion",
+        null=True,
+        related_name="removed_memberships",
+        on_delete=models.RESTRICT,
     )
 
     class Meta:


Test patch: