internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: pulp/pulpcore

Issue:
Update CI files for branch 3.21



Discussion:

Resolving patch:
diff --git a/pulpcore/app/openapigenerator.py b/pulpcore/app/openapigenerator.py
--- a/pulpcore/app/openapigenerator.py
+++ b/pulpcore/app/openapigenerator.py
@@ -255,10 +255,11 @@ def get_operation(self, operation_keys):
         produces = self.get_produces()
 
         multipart = ['multipart/form-data', 'application/x-www-form-urlencoded']
-        if 'file' in [param['type'] for param in self.get_request_body_parameters(multipart)]:
-            # automatically set the media type to form data if there's a file
-            # needed due to https://github.com/axnsan12/drf-yasg/issues/386
-            consumes = multipart
+        if self.method != 'GET':
+            if 'file' in [param['type'] for param in self.get_request_body_parameters(multipart)]:
+                # automatically set the media type to form data if there's a file
+                # needed due to https://github.com/axnsan12/drf-yasg/issues/386
+                consumes = multipart
 
         body = self.get_request_body_parameters(consumes)
         query = self.get_query_parameters()
diff --git a/pulpcore/app/viewsets/upload.py b/pulpcore/app/viewsets/upload.py
--- a/pulpcore/app/viewsets/upload.py
+++ b/pulpcore/app/viewsets/upload.py
@@ -4,7 +4,7 @@
 from drf_yasg.utils import swagger_auto_schema
 from drf_yasg.openapi import Parameter
 from rest_framework import mixins, serializers
-from rest_framework.decorators import detail_route
+from rest_framework.decorators import action
 from rest_framework.response import Response
 
 from pulpcore.app import tasks
@@ -94,7 +94,7 @@ def update(self, request, pk=None):
     @swagger_auto_schema(operation_summary="Finish an Upload",
                          request_body=UploadCommitSerializer,
                          responses={202: AsyncOperationResponseSerializer})
-    @detail_route(methods=('post',))
+    @action(detail=True, methods=['post'])
     def commit(self, request, pk):
         """
         Generates a Task to commit the upload and create an artifact
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
     'coreapi',
     'Django~=2.2',  # LTS version, switch only if we have a compelling reason to
     'django-filter',
-    'djangorestframework<3.10',
+    'djangorestframework',
     'djangorestframework-queryfields',
     'drf-nested-routers',
     'drf-yasg',


Test patch:
Repository: pulp/pulpcore

Issue:
[3.28] Fix files being deleted during file:// syncs
closes #4681


Discussion:

Resolving patch:
diff --git a/pulpcore/app/serializers/task.py b/pulpcore/app/serializers/task.py
--- a/pulpcore/app/serializers/task.py
+++ b/pulpcore/app/serializers/task.py
@@ -23,9 +23,11 @@ def to_representation(self, data):
                 return None
         except AttributeError:
             pass
-        request = self.context['request']
         viewset = get_viewset_for_model(data.content_object)
-        serializer = viewset.serializer_class(data.content_object, context={'request': request})
+
+        # serializer contains all serialized fields because we are passing
+        # 'None' to the request's context
+        serializer = viewset.serializer_class(data.content_object, context={'request': None})
         return serializer.data.get('_href')
 
     class Meta:


Test patch:
Repository: pulp/pulpcore

Issue:
Replace aioredis with redis-py
closes #3574
https://github.com/pulp/pulpcore/issues/3574

(cherry picked from commit 099e2eabc86e4bfe944c069d616cd172e1617b7e)


Discussion:

Resolving patch:
diff --git a/pulpcore/app/viewsets/custom_filters.py b/pulpcore/app/viewsets/custom_filters.py
--- a/pulpcore/app/viewsets/custom_filters.py
+++ b/pulpcore/app/viewsets/custom_filters.py
@@ -6,7 +6,7 @@
 from uuid import UUID
 
 from django.urls import Resolver404, resolve
-from django_filters import DateTimeFilter, Filter
+from django_filters import BaseInFilter, CharFilter, DateTimeFilter, Filter
 from django_filters.fields import IsoDateTimeField
 from rest_framework import serializers
 
@@ -190,3 +190,7 @@ def filter(self, qs, value):
 
         repo_version = self.get_repository_version(value)
         return qs.filter(pk__in=repo_version.removed())
+
+
+class CharInFilter(BaseInFilter, CharFilter):
+    pass


Test patch:
Repository: pulp/pulpcore

Issue:
Update CI files for branch 3.21

Update CI files for branch 3.21



Discussion:

Resolving patch:
diff --git a/pulpcore/app/serializers/publication.py b/pulpcore/app/serializers/publication.py
--- a/pulpcore/app/serializers/publication.py
+++ b/pulpcore/app/serializers/publication.py
@@ -220,9 +220,23 @@ class Meta:
     def validate(self, data):
         super().validate(data)
 
-        if 'repository' in data and 'repository_version' in data:
+        repository_in_data = 'repository' in data
+        repository_version_in_data = 'repository_version' in data
+        repository_in_instance = self.instance.repository if self.instance else None
+        repository_version_in_instance = self.instance.repository_version if self.instance else None
+
+        if repository_in_data and repository_version_in_data:
+            error = True
+        elif repository_in_data and repository_version_in_instance:
+            error = True
+        elif repository_in_instance and repository_version_in_data:
+            error = True
+        else:
+            error = False
+
+        if error:
             msg = _("The attributes 'repository' and 'repository_version' must be used"
-                    "exclusively.")
+                    " exclusively.")
             raise serializers.ValidationError(msg)
 
         return data


Test patch:
Repository: pulp/pulpcore

Issue:
Fix bug where Last-Modified header was being updated on duplicate package uploads
Fixes a bug where the Last-Modified header of a package stored in django-storages was being updated on duplicate uploads.

Closes #5149


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -4,21 +4,21 @@
     long_description = f.read()
 
 requirements = [
-    'coreapi',
-    'Django~=2.2',  # LTS version, switch only if we have a compelling reason to
-    'django-filter',
-    'djangorestframework',
-    'djangorestframework-queryfields',
-    'drf-nested-routers',
-    'drf-yasg',
-    'gunicorn',
+    'coreapi~=2.3.3',
+    'Django~=2.2.3',  # LTS version, switch only if we have a compelling reason to
+    'django-filter~=2.2.0',
+    'djangorestframework~=3.10.2',
+    'djangorestframework-queryfields~=1.0.0',
+    'drf-nested-routers~=0.91.0',
+    'drf-yasg~=1.16.1',
+    'gunicorn~=19.9.0',
     'packaging',  # until drf-yasg 1.16.2 is out https://github.com/axnsan12/drf-yasg/issues/412
-    'PyYAML',
-    'rq~=1.0',
-    'redis<3.2.0',
-    'setuptools',
-    'dynaconf~=2.0',
-    'whitenoise',
+    'PyYAML~=5.1.1',
+    'rq~=1.1.0',
+    'redis~=3.1.0',
+    'setuptools~=41.0.1',
+    'dynaconf~=2.0.3',
+    'whitenoise~=4.1.3',
 ]
 
 setup(


Test patch:
Repository: pulp/pulpcore

Issue:
file:// sync deletes files from directory
**Version**
Pulpcore 3.39

**Describe the bug**
When syncing file:// repositories, files are disappearing after the sync.

**To Reproduce**
1) Copy these two repositories to the FS:
  - https://github.com/Katello/katello/tree/master/test/fixtures/test_repos/file1
  - https://github.com/Katello/katello/tree/master/test/fixtures/test_repos/file2
2) Sync one, then the other
3) See that some files disappeared.
  - In my case, file2 lost every file except PULP_MANIFEST


**Expected behavior**
No files disappear.

**Additional context**
This also occurred with RPM content type files.



Discussion:

Resolving patch:
diff --git a/pulpcore/app/models/base.py b/pulpcore/app/models/base.py
--- a/pulpcore/app/models/base.py
+++ b/pulpcore/app/models/base.py
@@ -1,7 +1,9 @@
+from gettext import gettext as _
 import uuid
 
 from django.db import models
 from django.db.models import options
+from django.db.models.base import ModelBase
 
 
 class Model(models.Model):
@@ -35,7 +37,24 @@ def __repr__(self):
         return str(self)
 
 
-class MasterModel(Model):
+class MasterModelMeta(ModelBase):
+    def __new__(cls, name, bases, attrs, **kwargs):
+        """Override __new__ to set the default_related_name."""
+        if Model not in bases and MasterModel not in bases:  # Only affects "Detail" models.
+            meta = attrs.get("Meta")
+            default_related_name = getattr(
+                meta, "default_related_name", None)
+            abstract = getattr(meta, "abstract", None)
+
+            if not default_related_name and not abstract:
+                raise Exception(_("The 'default_related_name' option has not been set for "
+                                  "{class_name}").format(class_name=name))
+
+        new_class = super().__new__(cls, name, bases, attrs, **kwargs)
+        return new_class
+
+
+class MasterModel(Model, metaclass=MasterModelMeta):
     """Base model for the "Master" model in a "Master-Detail" relationship.
 
     Provides methods for casting down to detail types, back up to the master type,


Test patch:
Repository: pulp/pulpcore

Issue:
Update CI files for branch 3.39



Discussion:

Resolving patch:
diff --git a/pulpcore/app/serializers/task.py b/pulpcore/app/serializers/task.py
--- a/pulpcore/app/serializers/task.py
+++ b/pulpcore/app/serializers/task.py
@@ -58,7 +58,8 @@ class TaskSerializer(ModelSerializer):
                     "task."),
         read_only=True
     )
-    error = serializers.JSONField(
+    error = serializers.DictField(
+        child=serializers.JSONField(),
         help_text=_("A JSON Object of a fatal error encountered during the execution of this "
                     "task."),
         read_only=True


Test patch:
Repository: pulp/pulpcore

Issue:
Update CI files for branch 3.22



Discussion:

Resolving patch:
diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py
--- a/pulpcore/app/settings.py
+++ b/pulpcore/app/settings.py
@@ -223,7 +223,10 @@
     GLOBAL_ENV_FOR_DYNACONF='PULP',
     ENV_SWITCHER_FOR_DYNACONF='PULP_ENV',
     SETTINGS_MODULE_FOR_DYNACONF='/etc/pulp/settings.py',
-    INCLUDES_FOR_DYNACONF=['/etc/pulp/plugins/*'],
+    INCLUDES_FOR_DYNACONF=[
+        '{}.app.settings'.format(plugin_name)
+        for plugin_name in INSTALLED_PULP_PLUGINS
+    ],
     ENVVAR_FOR_DYNACONF='PULP_SETTINGS',
 )
 # HERE ENDS DYNACONF EXTENSION LOAD (No more code below this line)


Test patch:
Repository: pulp/pulpcore

Issue:
Make the import/export machinery compatible with `django-import-export>=4.0`
We should review the import/export machinery to understand how to update the `django-import-export` requirement.

#### Additional context
https://github.com/pulp/pulpcore/pull/5317


Discussion:

Resolving patch:
diff --git a/pulpcore/app/pagination.py b/pulpcore/app/pagination.py
--- a/pulpcore/app/pagination.py
+++ b/pulpcore/app/pagination.py
@@ -1,22 +1,6 @@
 from rest_framework import pagination
 
 
-class IDPagination(pagination.PageNumberPagination):
-    """
-    Paginate an API view naively, based on the ID of objects being iterated over.
-
-    This assumes that the objects being iterated over have an 'id' field, that the value of this
-    field is a int, and that the field is indexed.
-
-    This assumption should be True for all Models inheriting `pulpcore.app.models.Model`, the Pulp
-    base Model class.
-
-    """
-    ordering = 'id'
-    page_size_query_param = 'page_size'
-    max_page_size = 5000
-
-
 class NamePagination(pagination.PageNumberPagination):
     """
     Paginate an API view based on the value of the 'name' field of objects being iterated over.
diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py
--- a/pulpcore/app/settings.py
+++ b/pulpcore/app/settings.py
@@ -116,7 +116,7 @@
 REST_FRAMEWORK = {
     'URL_FIELD_NAME': '_href',
     'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',),
-    'DEFAULT_PAGINATION_CLASS': 'pulpcore.app.pagination.IDPagination',
+    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
     'PAGE_SIZE': 100,
     'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAuthenticated',),
     'DEFAULT_AUTHENTICATION_CLASSES': (


Test patch:
Repository: pulp/pulpcore

Issue:
[PR #5281/bd8cdc34 backport][3.51] Added csrf_token passing to django optional_logout
**This is a backport of PR #5281 as merged into main (bd8cdc343995c9b62e1ac32e012f71539beceabc).**

fixes: #5250


Discussion:

Resolving patch:
diff --git a/pulpcore/exceptions/base.py b/pulpcore/exceptions/base.py
--- a/pulpcore/exceptions/base.py
+++ b/pulpcore/exceptions/base.py
@@ -40,10 +40,7 @@ def exception_to_dict(exc, traceback=None):
     :return: dictionary representing the Exception
     :rtype: dict
     """
-    result = {'code': None, 'description': str(exc), 'traceback': traceback}
-    if isinstance(exc, PulpException):
-        result['code'] = exc.error_code
-    return result
+    return {'description': str(exc), 'traceback': traceback}
 
 
 class ResourceImmutableError(PulpException):


Test patch:
Repository: pulp/pulpcore

Issue:
[Backport-3.49] Fix import in wsgi preventing startup
Fixes #5189 



Discussion:

Resolving patch:
diff --git a/pulpcore/app/viewsets/upload.py b/pulpcore/app/viewsets/upload.py
--- a/pulpcore/app/viewsets/upload.py
+++ b/pulpcore/app/viewsets/upload.py
@@ -97,7 +97,7 @@ def update(self, request, pk=None):
     @action(detail=True, methods=['post'])
     def commit(self, request, pk):
         """
-        Generates a Task to commit the upload and create an artifact
+        Queues a Task that creates an Artifact, and the Upload gets deleted and cannot be re-used.
         """
         try:
             sha256 = request.data['sha256']


Test patch:
Repository: pulp/pulpcore

Issue:
[PR #5443/c7079c89 backport][3.54] Pin the pytest-redis requirement for unit tests
**This is a backport of PR #5443 as merged into main (c7079c891b87ecd9073260db5baa35826803f635).**

[noissue]


Discussion:

Resolving patch:
diff --git a/pulpcore/app/serializers/content.py b/pulpcore/app/serializers/content.py
--- a/pulpcore/app/serializers/content.py
+++ b/pulpcore/app/serializers/content.py
@@ -45,7 +45,7 @@ def __init__(self, *args, **kwargs):
 
         # If the content model has its own database field 'relative_path',
         # we should not mark the field write_only
-        if hasattr(self.Meta.model, 'relative_path'):
+        if hasattr(self.Meta.model, 'relative_path') and "relative_path" in self.fields:
             self.fields["relative_path"].write_only = False
 
     @transaction.atomic
@@ -57,7 +57,7 @@ def create(self, validated_data):
             validated_data (dict): Data to save to the database
         """
         artifact = validated_data.pop('artifact')
-        if self.fields["relative_path"].write_only:
+        if "relative_path" in self.fields and self.fields["relative_path"].write_only:
             relative_path = validated_data.pop('relative_path')
         else:
             relative_path = validated_data.get('relative_path')


Test patch:
Repository: pulp/pulpcore

Issue:
[PR #5443/c7079c89 backport][3.54] Pin the pytest-redis requirement for unit tests
**This is a backport of PR #5443 as merged into main (c7079c891b87ecd9073260db5baa35826803f635).**

[noissue]


Discussion:

Resolving patch:
diff --git a/pulpcore/app/serializers/content.py b/pulpcore/app/serializers/content.py
--- a/pulpcore/app/serializers/content.py
+++ b/pulpcore/app/serializers/content.py
@@ -57,7 +57,7 @@ def create(self, validated_data):
             validated_data (dict): Data to save to the database
         """
         artifact = validated_data.pop('artifact')
-        if "relative_path" in self.fields and self.fields["relative_path"].write_only:
+        if "relative_path" not in self.fields or self.fields["relative_path"].write_only:
             relative_path = validated_data.pop('relative_path')
         else:
             relative_path = validated_data.get('relative_path')


Test patch:
Repository: pulp/pulpcore

Issue:
Improve staging_docs landing page
What this PR does:
- Improve landing page layout
- Replace placeholder landing page cards with real content


Discussion:

Resolving patch:
diff --git a/pulpcore/app/openapigenerator.py b/pulpcore/app/openapigenerator.py
--- a/pulpcore/app/openapigenerator.py
+++ b/pulpcore/app/openapigenerator.py
@@ -2,6 +2,7 @@
 from collections import OrderedDict
 
 import uritemplate
+from django.utils.html import strip_tags
 from drf_yasg import openapi
 from drf_yasg.generators import OpenAPISchemaGenerator
 from drf_yasg.inspectors import SwaggerAutoSchema
@@ -283,6 +284,10 @@ def get_operation(self, operation_keys):
         else:
             operation_id = self.get_operation_id(operation_keys)
         summary, description = self.get_summary_and_description()
+
+        if "include_html" not in self.request.query_params:
+            description = strip_tags(description)
+
         security = self.get_security()
         assert security is None or isinstance(security, list), "security must be a list of " \
                                                                "security requirement objects"
diff --git a/pulpcore/app/settings.py b/pulpcore/app/settings.py
--- a/pulpcore/app/settings.py
+++ b/pulpcore/app/settings.py
@@ -215,6 +215,11 @@
     'DEFAULT_INFO': 'pulpcore.app.urls.api_info',
 }
 
+# have the docs url show field descriptions with html
+REDOC_SETTINGS = {
+    'SPEC_URL': '/pulp/api/v3/docs/?format=openapi&include_html=1',
+}
+
 # HERE STARTS DYNACONF EXTENSION LOAD (Keep at the very bottom of settings.py)
 # Read more at https://dynaconf.readthedocs.io/en/latest/guides/django.html
 import dynaconf  # noqa


Test patch:
Repository: pulp/pulpcore

Issue:
docs should not use underscores in http headers for auth
**Version**
any

**Describe the bug**
https://docs.pulpproject.org/pulpcore/authentication/webserver.html#webserver-auth-with-reverse-proxy says:
> With nginx providing authentication, all it can do is pass REMOTE_USER (or similar name) to the application webserver, i.e. gunicorn. You can pass the header as part of the proxy request in nginx with a config line like:
>     
>    proxy_set_header REMOTE_USER $remote_user;

But since gunicorn 22.0 (more precisely https://github.com/benoitc/gunicorn/commit/72b8970dbf2bf3444eb2e8b12aeff1a3d5922a9a/  https://github.com/benoitc/gunicorn/issues/2799) headers with underscores are forbidden by default.

If the docs would use a dash, so `proxy_set_header REMOTE-USER …` things would work :)

**Additional context**
Grant made me file this, and I did not want to post a screenshot of our colorful conversation ;)


Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/repository.py b/pulpcore/app/tasks/repository.py
--- a/pulpcore/app/tasks/repository.py
+++ b/pulpcore/app/tasks/repository.py
@@ -89,7 +89,10 @@ def add_and_remove(repository_pk, add_content_units, remove_content_units, base_
 
     if '*' in remove_content_units:
         latest = models.RepositoryVersion.latest(repository)
-        remove_content_units = latest.content.values_list('pk', flat=True)
+        if latest:
+            remove_content_units = latest.content.values_list('pk', flat=True)
+        else:
+            remove_content_units = []
 
     with models.RepositoryVersion.create(repository, base_version=base_version) as new_version:
         new_version.remove_content(models.Content.objects.filter(pk__in=remove_content_units))


Test patch:
Repository: pulp/pulpcore

Issue:
Fix content app not showing file size for 0 byte files
fixes: #5100


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -13,6 +13,7 @@
     'drf-yasg~=1.16.1',
     'gunicorn~=19.9.0',
     'packaging',  # until drf-yasg 1.16.2 is out https://github.com/axnsan12/drf-yasg/issues/412
+    'psycopg2-binary',
     'PyYAML~=5.1.1',
     'rq~=1.1.0',
     'redis~=3.1.0',
@@ -34,10 +35,6 @@
     url='http://www.pulpproject.org',
     python_requires='>=3.6',
     install_requires=requirements,
-    extras_require={
-        'postgres': ['psycopg2-binary'],
-        'mysql': ['mysqlclient']
-    },
     include_package_data=True,
     classifiers=(
         'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)',


Test patch:
Repository: pulp/pulpcore

Issue:
[PR #5007/075a37fd backport][3.45] Fix crud exporter test
**This is a backport of PR #5007 as merged into main (075a37fd5b68c09bddffb3205e2aa5c82e2bb549).**

Deletion task was not properly awaited.

[noissue]


Discussion:

Resolving patch:
diff --git a/pulpcore/app/models/content.py b/pulpcore/app/models/content.py
--- a/pulpcore/app/models/content.py
+++ b/pulpcore/app/models/content.py
@@ -244,6 +244,7 @@ class Content(MasterModel, QueryMixin):
         _artifacts (models.ManyToManyField): Artifacts related to Content through ContentArtifact
     """
     TYPE = 'content'
+    repo_key = ()
 
     _artifacts = models.ManyToManyField(Artifact, through='ContentArtifact')
 
diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py
--- a/pulpcore/app/models/repository.py
+++ b/pulpcore/app/models/repository.py
@@ -1,10 +1,14 @@
 """
 Repository related Django models.
 """
+from collections import defaultdict
 from contextlib import suppress
+from gettext import gettext as _
+import logging
 
 import django
 from django.db import models, transaction
+from django.db.models import Q
 from django.urls import reverse
 
 from pulpcore.app.util import get_view_name_for_model
@@ -15,6 +19,9 @@
 from .task import CreatedResource
 
 
+_logger = logging.getLogger(__name__)
+
+
 class Repository(Model):
     """
     Collection of content.
@@ -342,7 +349,12 @@ def next(self):
 
     def add_content(self, content):
         """
-        Add a content unit to this version.
+        Add a content unit to this version, and possibly remove duplcates in some situations.
+
+        In some situations plugin writers have configured new content to automatically remove other
+        content in the repository. For example, `pulp_file` content would conflict with a
+        `relative_path`. That is resolved by having the newer associated unit automatically remove
+        the other `pulp_file` content with the same `relative_path`.
 
         Args:
            content (django.db.models.QuerySet): Set of Content to add
@@ -351,9 +363,24 @@ def add_content(self, content):
             pulpcore.exception.ResourceImmutableError: if add_content is called on a
                 complete RepositoryVersion
         """
+
         if self.complete:
             raise ResourceImmutableError(self)
 
+        query_for_repo_duplicates_by_type = defaultdict(lambda: Q())
+        for item in content.all():
+            if item.repo_key == ():
+                continue
+            unit_q_dict = {
+                field: getattr(item, field) for field in item.repo_key
+            }
+            query_for_repo_duplicates_by_type[item._meta.model] |= Q(**unit_q_dict)
+
+        for model in query_for_repo_duplicates_by_type:
+            _logger.debug(_("Removing duplicates for type: {}".format(model)))
+            qs = model.objects.filter(query_for_repo_duplicates_by_type[model])
+            self.remove_content(qs)
+
         repo_content = []
         for content_pk in content.exclude(pk__in=self.content).values_list('pk', flat=True):
             repo_content.append(


Test patch: