internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: pulp/pulpcore

Issue:
Update CI files for branch 3.21
[noissue]


Discussion:

Resolving patch:
diff --git a/pulpcore/app/viewsets/base.py b/pulpcore/app/viewsets/base.py
--- a/pulpcore/app/viewsets/base.py
+++ b/pulpcore/app/viewsets/base.py
@@ -3,6 +3,7 @@
 from urllib.parse import urlparse
 
 from django.core.exceptions import FieldError, ValidationError
+from django.forms.utils import ErrorList
 from django.urls import Resolver404, resolve
 from django_filters.rest_framework import filterset
 from drf_yasg.utils import swagger_auto_schema
@@ -458,3 +459,20 @@ def filter_for_field(cls, field, name, lookup_expr):
                 field=name, expr=cls.LOOKUP_EXPR_TEXT[lookup_expr], value=val_word)
 
         return f
+
+    def is_valid(self, *args, **kwargs):
+        is_valid = super().is_valid(*args, **kwargs)
+        DEFAULT_FILTERS = [
+            "exclude_fields", "fields", "limit", "minimal", "offset", "page_size"
+        ]
+        for field in self.data.keys():
+            if field in DEFAULT_FILTERS:
+                continue
+
+            if field not in self.filters:
+                errors = self.form._errors.get("errors", ErrorList())
+                errors.extend(["Invalid Filter: '{field}'".format(field=field)])
+                self.form._errors["errors"] = errors
+                is_valid = False
+
+        return is_valid


Test patch:
Repository: pulp/pulpcore

Issue:
Update CI files for branch 3.21



Discussion:

Resolving patch:
diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py
--- a/pulpcore/app/models/repository.py
+++ b/pulpcore/app/models/repository.py
@@ -18,7 +18,7 @@
 
 from .base import MasterModel, Model
 from .content import Artifact, Content
-from .task import CreatedResource
+from .task import CreatedResource, Task
 
 
 _logger = logging.getLogger(__name__)
@@ -431,7 +431,7 @@ def create(cls, repository, base_version=None):
                 # now add any content that's in the base_version but not in version
                 version.add_content(base_version.content.exclude(pk__in=version.content))
 
-            if not repository.plugin_managed:
+            if Task.current and not repository.plugin_managed:
                 resource = CreatedResource(content_object=version)
                 resource.save()
             return version


Test patch:
Repository: pulp/pulpcore

Issue:
Improve the logic in the  ACSHandleStage
**Version**
main

**Describe the bug**
If there are ACS that point to the same content, pick the first RA instead of last

**To Reproduce**
Steps to reproduce the behavior:

**Expected behavior**
A clear and concise description of what you expected to happen.

**Additional context**
Add any other context about the problem here.  Please provide links to any previous discussions via Discourse or Bugzilla.



Discussion:

Resolving patch:
diff --git a/pulpcore/plugin/repo_version_utils.py b/pulpcore/plugin/repo_version_utils.py
--- a/pulpcore/plugin/repo_version_utils.py
+++ b/pulpcore/plugin/repo_version_utils.py
@@ -22,10 +22,6 @@ def remove_duplicates(repository_version):
     a `repo_key_fields` attribute with the field names to be compared. If all `repo_key_fields`
     contain the same value for two content units, they are considered "repository duplicates".
 
-    After instantiating `RemoveDuplicates` call it with the `run()` method and pass in the
-    :class:`~pulpcore.plugin.models.RepositoryVersion` to be checked and possibly modified as a
-    parameter to `run()`.
-
     Args:
         repository_version: The :class:`~pulpcore.plugin.models.RepositoryVersion` to be checked
             and possibly modified.


Test patch:
Repository: pulp/pulpcore

Issue:
File System Exporter should have an option to generate listing files
**Is your feature request related to a problem? Please describe.**
When Katello uses the File System exporter it does not generate the listing files in CDN directory structure. Katello does not directly have access to the pulp's content directories. Due to this it is not able to easily generate `Listing` files. It relies right now on the hammer export command being run from the same machine as pulp.

**Describe the solution you'd like**
Pulp exporter should ideally have an option or a flag to generate `listing` files when fs exporter is run.

**Describe alternatives you've considered**
Couple of things to consider here 
- This is only relevant to YUM/KS repos
- This probably only useful for certain Yum repos. But a flag in the exporter would take care of that.




Discussion:

Resolving patch:
diff --git a/pulpcore/app/serializers/content.py b/pulpcore/app/serializers/content.py
--- a/pulpcore/app/serializers/content.py
+++ b/pulpcore/app/serializers/content.py
@@ -1,4 +1,3 @@
-import hashlib
 from gettext import gettext as _
 
 from django.db import transaction
@@ -219,24 +218,24 @@ def validate(self, data):
                 _(f"Checksum algorithms {bad_algs} forbidden for this Pulp instance.")
             )
 
-        for algorithm in hashlib.algorithms_guaranteed:
-            if algorithm in models.Artifact.DIGEST_FIELDS:
-                digest = data["file"].hashers[algorithm].hexdigest()
-
-                if algorithm in data and digest != data[algorithm]:
-                    raise serializers.ValidationError(
-                        _("The %s checksum did not match.") % algorithm
-                    )
-                else:
-                    data[algorithm] = digest
-                if algorithm in models.Artifact.RELIABLE_DIGEST_FIELDS:
-                    validator = UniqueValidator(
-                        models.Artifact.objects.all(),
-                        message=_("{0} checksum must be unique.").format(algorithm),
-                    )
-                    validator.instance = None
-
-                    validator(digest, self.fields[algorithm])
+        for algorithm in reversed(models.Artifact.DIGEST_FIELDS):
+            digest = data["file"].hashers[algorithm].hexdigest()
+
+            if algorithm in data and digest != data[algorithm]:
+                raise serializers.ValidationError(_("The %s checksum did not match.") % algorithm)
+            else:
+                data[algorithm] = digest
+
+            if algorithm in models.Artifact.RELIABLE_DIGEST_FIELDS:
+                validator = UniqueValidator(
+                    models.Artifact.objects.all(),
+                    message=_("Artifact with {0} checksum of '{1}' already exists.").format(
+                        algorithm, digest
+                    ),
+                )
+                validator.instance = None
+                validator(digest, self.fields[algorithm])
+
         return data
 
     class Meta:


Test patch:
Repository: pulp/pulpcore

Issue:
Downloader retry logic is broken when retrying a partially written file
Author: @mdellweg (mdellweg)


Redmine Issue: 9673, https://pulp.plan.io/issues/9673

---

None





Discussion:

Resolving patch:
diff --git a/pulpcore/download/base.py b/pulpcore/download/base.py
--- a/pulpcore/download/base.py
+++ b/pulpcore/download/base.py
@@ -160,6 +160,7 @@ async def finalize(self):
         self._writer.flush()
         os.fsync(self._writer.fileno())
         self._writer.close()
+        self._writer = None
         self.validate_digests()
         self.validate_size()
 


Test patch:
Repository: pulp/pulpcore

Issue:
Retried downloads are not validating properly
When retrying a partial download, the size and digests need to be reset to validate.


Discussion:

Resolving patch:
diff --git a/pulpcore/download/base.py b/pulpcore/download/base.py
--- a/pulpcore/download/base.py
+++ b/pulpcore/download/base.py
@@ -103,7 +103,7 @@ def __init__(
             self.semaphore = semaphore
         else:
             self.semaphore = asyncio.Semaphore()  # This will always be acquired
-        self._digests = {n: pulp_hashlib.new(n) for n in Artifact.DIGEST_FIELDS}
+        self._digests = {}
         self._size = 0
         if self.expected_digests:
             if not set(self.expected_digests).intersection(set(Artifact.DIGEST_FIELDS)):
@@ -124,6 +124,8 @@ def _ensure_writer_has_open_file(self):
         if not self._writer:
             self._writer = tempfile.NamedTemporaryFile(dir=os.getcwd(), delete=False)
             self.path = self._writer.name
+            self._digests = {n: pulp_hashlib.new(n) for n in Artifact.DIGEST_FIELDS}
+            self._size = 0
 
     async def handle_data(self, data):
         """


Test patch:
Repository: pulp/pulpcore

Issue:
Deprecate unused custom_file_object in downloaders
This parameter is not properly handled by current code and must therefore be considered broken. Also it is unused.


Discussion:

Resolving patch:
diff --git a/pulpcore/download/base.py b/pulpcore/download/base.py
--- a/pulpcore/download/base.py
+++ b/pulpcore/download/base.py
@@ -7,6 +7,7 @@
 import tempfile
 
 from pulpcore.app import pulp_hashlib
+from pulpcore.app.loggers import deprecation_logger
 from pulpcore.app.models import Artifact
 from pulpcore.exceptions import (
     DigestValidationError,
@@ -94,6 +95,12 @@ def __init__(
             semaphore (asyncio.Semaphore): A semaphore the downloader must acquire before running.
                 Useful for limiting the number of outstanding downloaders in various ways.
         """
+        if custom_file_object:
+            deprecation_logger.warn(
+                "The 'custom_file_object' argument to 'BaseDownloader' is"
+                "deprecated and will be removed in pulpcore==3.20; stop using it."
+            )
+
         self.url = url
         self._writer = custom_file_object
         self.path = None


Test patch:
Repository: pulp/pulpcore

Issue:
Streaming range headers are calculated wrongly



Discussion:

Resolving patch:
diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py
--- a/pulpcore/content/handler.py
+++ b/pulpcore/content/handler.py
@@ -825,16 +825,17 @@ async def handle_response_headers(headers):
                     continue
 
                 if response.status == 206 and lower_name == "content-length":
-                    range_bytes = int(value)
+                    content_length = int(value)
                     start = 0 if range_start is None else range_start
-                    stop = range_bytes if range_stop is None else range_stop
+                    stop = content_length if range_stop is None else range_stop
 
-                    range_bytes = range_bytes - range_start
-                    range_bytes = range_bytes - (int(value) - stop)
+                    range_bytes = stop - start
                     response.headers[name] = str(range_bytes)
 
+                    # aiohttp adds a 1 to the range.stop compared to http headers (including) to
+                    # match python array adressing (exclusive)
                     response.headers["Content-Range"] = "bytes {0}-{1}/{2}".format(
-                        start, stop - start + 1, int(value)
+                        start, stop - 1, content_length
                     )
                     continue
 


Test patch:
Repository: pulp/pulpcore

Issue:
Streaming range headers are calculated wrongly



Discussion:

Resolving patch:
diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py
--- a/pulpcore/content/handler.py
+++ b/pulpcore/content/handler.py
@@ -837,16 +837,17 @@ async def handle_response_headers(headers):
                     continue
 
                 if response.status == 206 and lower_name == "content-length":
-                    range_bytes = int(value)
+                    content_length = int(value)
                     start = 0 if range_start is None else range_start
-                    stop = range_bytes if range_stop is None else range_stop
+                    stop = content_length if range_stop is None else range_stop
 
-                    range_bytes = range_bytes - range_start
-                    range_bytes = range_bytes - (int(value) - stop)
+                    range_bytes = stop - start
                     response.headers[name] = str(range_bytes)
 
+                    # aiohttp adds a 1 to the range.stop compared to http headers (including) to
+                    # match python array adressing (exclusive)
                     response.headers["Content-Range"] = "bytes {0}-{1}/{2}".format(
-                        start, stop - start + 1, int(value)
+                        start, stop - 1, content_length
                     )
                     continue
 


Test patch:
Repository: pulp/pulpcore

Issue:
Canceling tasks never picked up by a worker marks them failed
Code investigation shows, we are doing the right thing for the right reason, but reported the wrong thing with the wrong reason.


Discussion:

Resolving patch:
diff --git a/pulpcore/tasking/pulpcore_worker.py b/pulpcore/tasking/pulpcore_worker.py
--- a/pulpcore/tasking/pulpcore_worker.py
+++ b/pulpcore/tasking/pulpcore_worker.py
@@ -194,6 +194,13 @@ def iter_tasks(self):
                     # The lock will be automatically be released at the end of the block
                     # Check if someone else changed the task before we got the lock
                     task.refresh_from_db()
+                    if task.state == TASK_STATES.CANCELING and task.worker is None:
+                        # No worker picked this task up before being canceled
+                        if self.cancel_abandoned_task(task, TASK_STATES.CANCELED):
+                            # Continue looking for the next task
+                            # without considering this tasks resources
+                            # as we just released them
+                            continue
                     if task.state in [TASK_STATES.RUNNING, TASK_STATES.CANCELING]:
                         # A running task without a lock must be abandoned
                         if self.cancel_abandoned_task(


Test patch:
Repository: pulp/pulpcore

Issue:
Circular import exists when using custom user model
This currently occurs when a plugin using a custom user model depends on the Role model [here](https://github.com/pulp/pulpcore/blob/main/pulpcore/app/models/__init__.py#L61).


Discussion:

Resolving patch:
diff --git a/pulpcore/app/models/__init__.py b/pulpcore/app/models/__init__.py
--- a/pulpcore/app/models/__init__.py
+++ b/pulpcore/app/models/__init__.py
@@ -59,6 +59,7 @@
 )
 
 # This can lead to circular imports with a custom user model depending on this very module
+# Moved to plugin/models/role.py to avoid the circular import.
 # from .role import (  # noqa
 #     GroupRole,
 #     Role,
diff --git a/pulpcore/plugin/models/role.py b/pulpcore/plugin/models/role.py
new file mode 100644
--- /dev/null
+++ b/pulpcore/plugin/models/role.py
@@ -0,0 +1,8 @@
+# Models are exposed selectively in the versioned plugin API.
+# Any models defined in the pulpcore.plugin namespace should probably be proxy models.
+
+from pulpcore.app.models.role import (  # noqa
+    GroupRole,
+    Role,
+    UserRole,
+)


Test patch:
Repository: pulp/pulpcore

Issue:
PulpImporter assumes tempfiles can always go to /tmp
This issue is a copy of https://pulp.plan.io/issues/8610 , to allow us to backport the fix from core/3.17 into 14/15/16 correctly.

**Version**
core/3.14+

**Describe the bug**
importer.pulp_import uses tempfile.TemporaryDirectory() in places like this:

https://github.com/pulp/pulpcore/blob/master/pulpcore/app/tasks/importer.py#L118

If your /tmp is small, and your export is Large, this can cause Bad Things to happen.

We should perhas set dir= to the workers work-directory?




Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py
--- a/pulpcore/app/tasks/importer.py
+++ b/pulpcore/app/tasks/importer.py
@@ -373,7 +373,7 @@ def validate_and_assemble(toc_filename):
     )
     CreatedResource.objects.create(content_object=the_import)
 
-    with tempfile.TemporaryDirectory() as temp_dir:
+    with tempfile.TemporaryDirectory(dir=".") as temp_dir:
         with tarfile.open(path, "r:gz") as tar:
             tar.extractall(path=temp_dir)
 


Test patch:
Repository: pulp/pulpcore

Issue:
PulpImporter assumes tempfiles can always go to /tmp
This issue is a copy of https://pulp.plan.io/issues/8610 , to allow us to backport the fix from core/3.17 into 14/15/16 correctly.

**Version**
core/3.14+

**Describe the bug**
importer.pulp_import uses tempfile.TemporaryDirectory() in places like this:

https://github.com/pulp/pulpcore/blob/master/pulpcore/app/tasks/importer.py#L118

If your /tmp is small, and your export is Large, this can cause Bad Things to happen.

We should perhas set dir= to the workers work-directory?




Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py
--- a/pulpcore/app/tasks/importer.py
+++ b/pulpcore/app/tasks/importer.py
@@ -377,7 +377,7 @@ def validate_and_assemble(toc_filename):
     current_task.refresh_from_db()
     CreatedResource.objects.create(content_object=task_group)
 
-    with tempfile.TemporaryDirectory() as temp_dir:
+    with tempfile.TemporaryDirectory(dir=".") as temp_dir:
         with tarfile.open(path, "r:gz") as tar:
             tar.extractall(path=temp_dir)
 


Test patch:
Repository: pulp/pulpcore

Issue:
PulpImporter assumes tempfiles can always go to /tmp
This issue is a copy of https://pulp.plan.io/issues/8610 , to allow us to backport the fix from core/3.17 into 14/15/16 correctly.

**Version**
core/3.14+

**Describe the bug**
importer.pulp_import uses tempfile.TemporaryDirectory() in places like this:

https://github.com/pulp/pulpcore/blob/master/pulpcore/app/tasks/importer.py#L118

If your /tmp is small, and your export is Large, this can cause Bad Things to happen.

We should perhas set dir= to the workers work-directory?




Discussion:

Resolving patch:
diff --git a/pulpcore/app/tasks/importer.py b/pulpcore/app/tasks/importer.py
--- a/pulpcore/app/tasks/importer.py
+++ b/pulpcore/app/tasks/importer.py
@@ -378,7 +378,7 @@ def validate_and_assemble(toc_filename):
     current_task.refresh_from_db()
     CreatedResource.objects.create(content_object=task_group)
 
-    with tempfile.TemporaryDirectory() as temp_dir:
+    with tempfile.TemporaryDirectory(dir=".") as temp_dir:
         with tarfile.open(path, "r:gz") as tar:
             tar.extractall(path=temp_dir)
 


Test patch:
Repository: pulp/pulpcore

Issue:
Expose PulpRemoteUserAuthentication to plugin writers
This will allow plugin writers to take a rest_framework remote authentication class into account when building protected endpoints.


Discussion:

Resolving patch:
diff --git a/pulpcore/plugin/authentication/__init__.py b/pulpcore/plugin/authentication/__init__.py
new file mode 100644
--- /dev/null
+++ b/pulpcore/plugin/authentication/__init__.py
@@ -0,0 +1 @@
+from pulpcore.app.authentication import PulpRemoteUserAuthentication  # noqa


Test patch:
Repository: pulp/pulpcore

Issue:
Filter user roles by description
**Is your feature request related to a problem? Please describe.**
We would like to give the user the power to filter roles by name and by description. The pulpcore does not allow filtering by description yet.

**Describe the solution you'd like**
Allow filtering by description. A case insensitive contains is enough for this field.

**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.

**Additional context**
Connected issue: issues.redhat.com/browse/AAH-1409



Discussion:

Resolving patch:
diff --git a/pulpcore/app/viewsets/user.py b/pulpcore/app/viewsets/user.py
--- a/pulpcore/app/viewsets/user.py
+++ b/pulpcore/app/viewsets/user.py
@@ -481,6 +481,7 @@ class Meta:
         model = Role
         fields = {
             "name": NAME_FILTER_OPTIONS,
+            "description": ["exact", "iexact", "icontains", "contains"],
             "locked": ["exact"],
         }
 


Test patch:
Repository: pulp/pulpcore

Issue:
Clean up TaskReservedResources/task-table at migration to new-tasking-system
See https://bugzilla.redhat.com/show_bug.cgi?id=2031154 for details.

Migration that needs to be updated to purge taskreservedresource entries: 0064_add_new_style_task_columns.py

This wants to be cherrypicked into 3.14/15/16 (after which the offending table no longer exists)


Discussion:
Add core_task truncation - makes migration faster, may cascade-delete core_taskreservedresource as well?

Resolving patch:
diff --git a/pulpcore/app/migrations/0064_add_new_style_task_columns.py b/pulpcore/app/migrations/0064_add_new_style_task_columns.py
--- a/pulpcore/app/migrations/0064_add_new_style_task_columns.py
+++ b/pulpcore/app/migrations/0064_add_new_style_task_columns.py
@@ -4,16 +4,34 @@
 import django.contrib.postgres.fields.jsonb
 from django.db import migrations, models
 
+TASK_BATCH_SIZE = 1000
+
 
 def copy_reserved_resources_record(apps, schema_editor):
     Task = apps.get_model('core', 'Task')
-    for task in Task.objects.iterator():
+
+    # Update _reserved_resource_record for all tasks, 1000 tasks at a time.
+    # When we hit 1K tasks, go to the db for the batch.
+    # Make sure to update the final batch!
+    tasks = []
+    for task in Task.objects.iterator(chunk_size=TASK_BATCH_SIZE):
         task._reserved_resources_record = list(task.reserved_resources_record.values_list('resource', flat=True))
-        task.save()
+        tasks.append(task)
+        if len(tasks) == TASK_BATCH_SIZE:
+            Task.objects.bulk_update(tasks, ["_reserved_resources_record"])
+            tasks.clear()
+
+    # Update last set of tasks
+    if len(tasks) > 0:
+        Task.objects.bulk_update(tasks, ["_reserved_resources_record"])
+
 
+def purge_reservedresources(apps, schema_editor):
+    TaskReservedResource = apps.get_model('core', 'TaskReservedResource')
+    TaskReservedResource.objects.all().delete()
 
-def noop(apps, schema_editor):
-    pass
+    ReservedResource = apps.get_model('core', 'ReservedResource')
+    ReservedResource.objects.all().delete()
 
 
 class Migration(migrations.Migration):
@@ -23,6 +41,12 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        # Purge any ReservedResource entries - if there are any, they're orphans
+        migrations.RunPython(
+            code=purge_reservedresources,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        # Update entities for the new task-system
         migrations.AddField(
             model_name='task',
             name='args',
@@ -59,7 +83,7 @@ class Migration(migrations.Migration):
         ),
         migrations.RunPython(
             code=copy_reserved_resources_record,
-            reverse_code=noop,
+            reverse_code=migrations.RunPython.noop,
         ),
         migrations.RemoveField(
             model_name='taskreservedresourcerecord',
@@ -80,4 +104,5 @@ class Migration(migrations.Migration):
             old_name='_reserved_resources_record',
             new_name='reserved_resources_record',
         ),
+
     ]


Test patch: