Repository: pulp/pulpcore

Issue:
As a user I want to be able to rotate my encryption keys and rekey my informations
Author: spredzy (spredzy)


Redmine Issue: 9397, https://pulp.plan.io/issues/9397

---

Pulp 3.15 brings the support for encrypting fields in the DB. This is a great step toward better security practices. Thanks team for that.

In order to go a step further with security best-practices, I would like to be able to rotate my keys periodically, and hence rekey my data.

As it's stand today I haven't see a way to do this.





Discussion:
From: @mdellweg (mdellweg)
Date: 2021-09-15T08:35:20Z

---

We should provide a `pulpcore-manager` command to replace the key.

Also there should be a strategy how to rekey in a clustered environment.
From: daviddavis (daviddavis)
Date: 2021-09-20T12:06:21Z

---

mdellweg wrote:
> We should provide a `pulpcore-manager` command to replace the key.

+1. How will the command work? What options will it take?
 
> Also there should be a strategy how to rekey in a clustered environment.

Maybe we should file another issue for this? Or turn this into an epic with subtasks?

From: @bmbouter (bmbouter)
Date: 2021-09-20T14:11:10Z

---

+1 to an epic with subtasks.

Also just to state it: It's both the changing of the key and the decrypt-re-encrypt of data in the database. Also here are two things I'm thinking about:

1) In clustered installs, how do we ensure the keys are distributed to all the nodes (which will need the private key) yet ensure the decrypt re-encrypt will only happen exactly once?

2) What happens if an OOM or power loss occurs on whatever node is being run halfway through? Since the data is encrypted, we have to be extremely careful that this is bulletproof.
From: @fao89 (fao89)
Date: 2021-09-21T14:40:05Z

---

I believe this may involve pulpcore, pulp_installer and pulp-operator work
This issue has been marked 'stale' due to lack of recent activity. If there is no further activity, the issue will be closed in another 30 days. Thank you for your contribution!
This issue is no longer marked for closure.

Resolving patch:
diff --git a/pulpcore/app/management/commands/rotate-db-key.py b/pulpcore/app/management/commands/rotate-db-key.py
new file mode 100644
--- /dev/null
+++ b/pulpcore/app/management/commands/rotate-db-key.py
@@ -0,0 +1,72 @@
+from contextlib import suppress
+from gettext import gettext as _
+
+from django.apps import apps
+from django.core.management import BaseCommand
+from django.db import connection, transaction
+
+from pulpcore.app.models import MasterModel
+from pulpcore.app.models.fields import EncryptedTextField, EncryptedJSONField
+
+
+class DryRun(Exception):
+    pass
+
+
+class Command(BaseCommand):
+    """
+    Django management command for db key rotation.
+    """
+
+    help = _(
+        "Rotate the db encryption key. "
+        "This command will re-encrypt all values in instances of EncryptedTextField and "
+        "EncryptedJSONField with the first key in the file refereced by "
+        "settings.DB_ENCRYPTION_KEY. You need to make sure that all running instances of the "
+        "application already loaded this key for proper functioning. Refer to the docs for zero "
+        "downtime key rotation."
+        "It is safe to abort and resume or rerun this operation."
+    )
+
+    def add_arguments(self, parser):
+        """Set up arguments."""
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help=_("Don't modify anything."),
+        )
+
+    def handle(self, *args, **options):
+        dry_run = options["dry_run"]
+
+        for model in apps.get_models():
+            if issubclass(model, MasterModel) and model._meta.master_model is None:
+                # This is a master model, and we will handle all it's descendents.
+                continue
+            field_names = [
+                field.name
+                for field in model._meta.get_fields()
+                if isinstance(field, (EncryptedTextField, EncryptedJSONField))
+            ]
+            if field_names:
+                print(
+                    _("Updating {fields} on {model}.").format(
+                        model=model.__name__, fields=",".join(field_names)
+                    )
+                )
+                exclude_filters = {f"{field_name}": None for field_name in field_names}
+                qs = model.objects.exclude(**exclude_filters).only(*field_names)
+                with suppress(DryRun), transaction.atomic():
+                    batch = []
+                    for item in qs.iterator():
+                        batch.append(item)
+                        if len(batch) >= 1024:
+                            model.objects.bulk_update(batch, field_names)
+                            batch = []
+                    if batch:
+                        model.objects.bulk_update(batch, field_names)
+                        batch = []
+                    if dry_run:
+                        with connection.cursor() as cursor:
+                            cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+                        raise DryRun()
diff --git a/pulpcore/app/models/fields.py b/pulpcore/app/models/fields.py
--- a/pulpcore/app/models/fields.py
+++ b/pulpcore/app/models/fields.py
@@ -1,14 +1,14 @@
 import logging
 import os
 from gettext import gettext as _
+from functools import lru_cache
 
-from cryptography.fernet import Fernet
+from cryptography.fernet import Fernet, MultiFernet
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db.models import Lookup, FileField, JSONField
 from django.db.models.fields import Field, TextField
 from django.utils.encoding import force_bytes, force_str
-from django.utils.functional import cached_property
 
 
 from pulpcore.app.files import TemporaryDownloadedFile
@@ -16,6 +16,14 @@
 _logger = logging.getLogger(__name__)
 
 
+@lru_cache()
+def _fernet():
+    # Cache the enryption keys once per application.
+    _logger.debug(f"Loading encryption key from {settings.DB_ENCRYPTION_KEY}")
+    with open(settings.DB_ENCRYPTION_KEY, "rb") as key_file:
+        return MultiFernet([Fernet(key) for key in key_file.readlines()])
+
+
 class ArtifactFileField(FileField):
     """
     A custom FileField that always saves files to location specified by 'upload_to'.
@@ -88,20 +96,14 @@ def __init__(self, *args, **kwargs):
             raise ImproperlyConfigured("EncryptedTextField does not support db_index=True.")
         super().__init__(*args, **kwargs)
 
-    @cached_property
-    def _fernet(self):
-        _logger.debug(f"Loading encryption key from {settings.DB_ENCRYPTION_KEY}")
-        with open(settings.DB_ENCRYPTION_KEY, "rb") as key_file:
-            return Fernet(key_file.read())
-
     def get_db_prep_save(self, value, connection):
         value = super().get_db_prep_save(value, connection)
         if value is not None:
-            return force_str(self._fernet.encrypt(force_bytes(value)))
+            return force_str(_fernet().encrypt(force_bytes(value)))
 
     def from_db_value(self, value, expression, connection):
         if value is not None:
-            return force_str(self._fernet.decrypt(force_bytes(value)))
+            return force_str(_fernet().decrypt(force_bytes(value)))
 
 
 class EncryptedJSONField(JSONField):
@@ -116,19 +118,13 @@ def __init__(self, *args, **kwargs):
             raise ImproperlyConfigured("EncryptedJSONField does not support db_index=True.")
         super().__init__(*args, **kwargs)
 
-    @cached_property
-    def _fernet(self):
-        _logger.debug(f"Loading encryption key from {settings.DB_ENCRYPTION_KEY}")
-        with open(settings.DB_ENCRYPTION_KEY, "rb") as key_file:
-            return Fernet(key_file.read())
-
     def encrypt(self, value):
         if isinstance(value, dict):
             return {k: self.encrypt(v) for k, v in value.items()}
         elif isinstance(value, (list, tuple, set)):
             return [self.encrypt(v) for v in value]
 
-        return force_str(self._fernet.encrypt(force_bytes(repr(value))))
+        return force_str(_fernet().encrypt(force_bytes(repr(value))))
 
     def decrypt(self, value):
         if isinstance(value, dict):
@@ -136,7 +132,7 @@ def decrypt(self, value):
         elif isinstance(value, (list, tuple, set)):
             return [self.decrypt(v) for v in value]
 
-        return eval(force_str(self._fernet.decrypt(force_bytes(value))))
+        return eval(force_str(_fernet().decrypt(force_bytes(value))))
 
     def get_db_prep_save(self, value, connection):
         value = self.encrypt(value)


Test patch:
diff --git a/pulpcore/tests/unit/models/test_remote.py b/pulpcore/tests/unit/models/test_remote.py
--- a/pulpcore/tests/unit/models/test_remote.py
+++ b/pulpcore/tests/unit/models/test_remote.py
@@ -5,7 +5,7 @@
 from django.test import TestCase
 
 from pulpcore.app.models import Remote
-from pulpcore.app.models.fields import EncryptedTextField
+from pulpcore.app.models.fields import _fernet, EncryptedTextField
 
 
 class RemoteTestCase(TestCase):
@@ -15,6 +15,7 @@ def setUp(self):
     def tearDown(self):
         if self.remote:
             self.remote.delete()
+        _fernet.cache_clear()
 
     @patch(
         "pulpcore.app.models.fields.open",
@@ -22,6 +23,7 @@ def tearDown(self):
         read_data=b"hPCIFQV/upbvPRsEpgS7W32XdFA2EQgXnMtyNAekebQ=",
     )
     def test_encrypted_proxy_password(self, mock_file):
+        _fernet.cache_clear()
         self.remote = Remote(name=uuid4(), proxy_password="test")
         self.remote.save()
         self.assertEqual(Remote.objects.get(pk=self.remote.pk).proxy_password, "test")
@@ -35,4 +37,4 @@ def test_encrypted_proxy_password(self, mock_file):
             proxy_password = EncryptedTextField().from_db_value(db_proxy_password, None, connection)
             self.assertNotEqual(db_proxy_password, "test")
             self.assertEqual(proxy_password, "test")
-            self.assertEqual(mock_file.call_count, 2)
+            self.assertEqual(mock_file.call_count, 1)