Repository: pulp/pulpcore

Issue:
Remove unencrypted task arguments



Discussion:

Resolving patch:
diff --git a/pulpcore/app/migrations/0114_remove_task_args_remove_task_kwargs.py b/pulpcore/app/migrations/0114_remove_task_args_remove_task_kwargs.py
new file mode 100644
--- /dev/null
+++ b/pulpcore/app/migrations/0114_remove_task_args_remove_task_kwargs.py
@@ -0,0 +1,48 @@
+# Generated by Django 4.2.4 on 2023-09-11 13:26
+
+from django.db import migrations
+from django.db.models import Q
+from packaging.version import parse as parse_version
+
+from pulpcore.migrations import RequireVersion
+
+
+def encrypt_task_args_up(apps, schema_editor):
+    Task = apps.get_model("core", "Task")
+    batch = []
+    for task in Task.objects.filter(Q(args__is_null=False) | Q(kwargs__is_null=False)).iterator():
+        task.enc_args = task.args
+        task.enc_kwargs = task.kwargs
+        coreversion = task.versions.get("core")
+        if coreversion is None or parse_version(coreversion) < parse_version("3.34.0"):
+            task.versions["core"] = "3.34.0"
+
+        batch.append(task)
+        if len(batch) >= 500:
+            Task.bulk_update(batch, fields=["enc_args", "enc_kwargs", "versions"])
+            batch = []
+    if batch:
+        Task.bulk_update(batch, fields=["enc_args", "enc_kwargs", "versions"])
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0113_headercontentguard"),
+    ]
+
+    operations = [
+        RequireVersion("core", "3.34.0"),
+        migrations.RunPython(
+            code=encrypt_task_args_up,
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        ),
+        migrations.RemoveField(
+            model_name="task",
+            name="args",
+        ),
+        migrations.RemoveField(
+            model_name="task",
+            name="kwargs",
+        ),
+    ]
diff --git a/pulpcore/app/models/task.py b/pulpcore/app/models/task.py
--- a/pulpcore/app/models/task.py
+++ b/pulpcore/app/models/task.py
@@ -83,12 +83,6 @@ class Task(BaseModel, AutoAddObjPermsMixin):
 
     error = models.JSONField(null=True)
 
-    # These fields should finally be removed in 3.40 by a migration that checks all components are
-    # running at least <version this landed> then copies their values to enc_(kw)args and adds
-    # pulpcore >= <version this landed> in the modified tasks version_requirements.
-    args = models.JSONField(null=True, encoder=DjangoJSONEncoder)
-    kwargs = models.JSONField(null=True, encoder=DjangoJSONEncoder)
-
     enc_args = EncryptedJSONField(null=True, encoder=DjangoJSONEncoder)
     enc_kwargs = EncryptedJSONField(null=True, encoder=DjangoJSONEncoder)
 
diff --git a/pulpcore/tasking/tasks.py b/pulpcore/tasking/tasks.py
--- a/pulpcore/tasking/tasks.py
+++ b/pulpcore/tasking/tasks.py
@@ -56,8 +56,8 @@ def _execute_task(task):
         module_name, function_name = task.name.rsplit(".", 1)
         module = importlib.import_module(module_name)
         func = getattr(module, function_name)
-        args = task.enc_args or task.args or ()
-        kwargs = task.enc_kwargs or task.kwargs or {}
+        args = task.enc_args or ()
+        kwargs = task.enc_kwargs or {}
         result = func(*args, **kwargs)
         if asyncio.iscoroutine(result):
             _logger.debug(_("Task is coroutine %s"), task.pk)


Test patch:
diff --git a/pulpcore/tests/functional/__init__.py b/pulpcore/tests/functional/__init__.py
--- a/pulpcore/tests/functional/__init__.py
+++ b/pulpcore/tests/functional/__init__.py
@@ -1000,17 +1000,16 @@ def _add_to_cleanup(api_client, pulp_href):
     delete_task_hrefs = []
     # Delete newest items first to avoid dependency lockups
     for api_client, pulp_href in reversed(obj_refs):
-        try:
-            task_url = api_client.delete(pulp_href).task
-            delete_task_hrefs.append(task_url)
-        except Exception:
+        with suppress(Exception):
             # There was no delete task for this unit or the unit may already have been deleted.
             # Also we can never be sure which one is the right ApiException to catch.
-            pass
+            task_url = api_client.delete(pulp_href).task
+            delete_task_hrefs.append(task_url)
 
     for deleted_task_href in delete_task_hrefs:
-        with suppress(ApiException):
+        with suppress(Exception):
             # The task itself may be gone at this point (e.g. by being part of a deleted domain).
+            # Also we can never be sure which one is the right ApiException to catch.
             monitor_task(deleted_task_href)
 
 
diff --git a/pulpcore/tests/functional/api/pulp_file/test_domains.py b/pulpcore/tests/functional/api/pulp_file/test_domains.py
--- a/pulpcore/tests/functional/api/pulp_file/test_domains.py
+++ b/pulpcore/tests/functional/api/pulp_file/test_domains.py
@@ -8,8 +8,7 @@
 from pulpcore.tests.functional.utils import generate_iso, download_file
 
 
-if not settings.DOMAIN_ENABLED:
-    pytest.skip("Domains not enabled.", allow_module_level=True)
+pytestmark = pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domains not enabled.")
 
 
 @pytest.mark.parallel
diff --git a/pulpcore/tests/functional/api/pulp_file/test_pulp_export.py b/pulpcore/tests/functional/api/pulp_file/test_pulp_export.py
--- a/pulpcore/tests/functional/api/pulp_file/test_pulp_export.py
+++ b/pulpcore/tests/functional/api/pulp_file/test_pulp_export.py
@@ -6,16 +6,14 @@
 from pulpcore.constants import TASK_STATES
 
 
-if settings.DOMAIN_ENABLED:
-    pytest.skip("Domains do not support export.", allow_module_level=True)
-
-
-if "/tmp" not in settings.ALLOWED_EXPORT_PATHS:
-    pytest.skip(
-        "Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS "
+pytestmark = [
+    pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support export."),
+    pytest.mark.skipif(
+        "/tmp" not in settings.ALLOWED_EXPORT_PATHS,
+        reason="Cannot run export-tests unless /tmp is in ALLOWED_EXPORT_PATHS "
         f"({settings.ALLOWED_EXPORT_PATHS}).",
-        allow_module_level=True,
-    )
+    ),
+]
 
 
 @pytest.fixture
diff --git a/pulpcore/tests/functional/api/test_replication.py b/pulpcore/tests/functional/api/test_replication.py
--- a/pulpcore/tests/functional/api/test_replication.py
+++ b/pulpcore/tests/functional/api/test_replication.py
@@ -7,7 +7,7 @@
 from pulpcore.tests.functional.utils import PulpTaskGroupError
 
 
-@pytest.mark.parallel
+# @pytest.mark.parallel
 def test_replication(
     domain_factory,
     bindings_cfg,
@@ -41,7 +41,7 @@ def test_replication(
         assert task.state == "completed"
 
 
-@pytest.mark.parallel
+# @pytest.mark.parallel
 def test_replication_with_wrong_ca_cert(
     domain_factory,
     bindings_cfg,
diff --git a/pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py b/pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py
--- a/pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py
+++ b/pulpcore/tests/functional/api/using_plugin/test_filesystemexport.py
@@ -15,8 +15,7 @@
 NUM_REPOS = 1
 NUM_EXPORTERS = 4
 
-if settings.DOMAIN_ENABLED:
-    pytest.skip("Domains do not support export.", allow_module_level=True)
+pytestmark = pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support export.")
 
 
 @pytest.fixture
diff --git a/pulpcore/tests/functional/api/using_plugin/test_pulpimport.py b/pulpcore/tests/functional/api/using_plugin/test_pulpimport.py
--- a/pulpcore/tests/functional/api/using_plugin/test_pulpimport.py
+++ b/pulpcore/tests/functional/api/using_plugin/test_pulpimport.py
@@ -20,17 +20,15 @@
 NUM_REPOS = 2
 
 
-if settings.DOMAIN_ENABLED:
-    pytest.skip("Domains do not support import.", allow_module_level=True)
-
-
-if "/tmp" not in settings.ALLOWED_IMPORT_PATHS:
-    raise pytest.skip(
-        "Cannot run import-tests unless /tmp is in ALLOWED_IMPORT_PATHS ({}).".format(
+pytestmark = [
+    pytest.mark.skipif(settings.DOMAIN_ENABLED, reason="Domains do not support import."),
+    pytest.mark.skipif(
+        "/tmp" not in settings.ALLOWED_IMPORT_PATHS,
+        reason="Cannot run import-tests unless /tmp is in ALLOWED_IMPORT_PATHS ({}).".format(
             settings.ALLOWED_IMPORT_PATHS
         ),
-        allow_module_level=True,
-    )
+    ),
+]
 
 
 @pytest.fixture
diff --git a/pulpcore/tests/functional/api/using_plugin/test_repair.py b/pulpcore/tests/functional/api/using_plugin/test_repair.py
--- a/pulpcore/tests/functional/api/using_plugin/test_repair.py
+++ b/pulpcore/tests/functional/api/using_plugin/test_repair.py
@@ -16,13 +16,12 @@
     "pulpcore.app.models.storage.FileSystem",
 ]
 
-if settings.DEFAULT_FILE_STORAGE not in SUPPORTED_STORAGE_FRAMEWORKS:
-    raise pytest.skip(
-        "Cannot simulate bit-rot on this storage platform ({}).".format(
-            settings.DEFAULT_FILE_STORAGE
-        ),
-        allow_module_level=True,
-    )
+pytestmark = pytest.mark.skipif(
+    settings.DEFAULT_FILE_STORAGE not in SUPPORTED_STORAGE_FRAMEWORKS,
+    reason="Cannot simulate bit-rot on this storage platform ({}).".format(
+        settings.DEFAULT_FILE_STORAGE
+    ),
+)
 
 
 @pytest.fixture