Repository: pulp/pulpcore

Issue:
Deleting repository versions can loose track of later content deletion
**Version**
pulpcore 3.18

**Describe the bug**
When deleting a repository version that deletes a content that is added back in the subsequent version, but deleted again in a later version that deletion in the later version is lost.

**To Reproduce**
Steps to reproduce the behavior:
```bash
#!/bin/bash

set -eu

pulp file repository destroy --name test_delete_versions || true
pulp file repository create --name test_delete_versions

for NAME in "aaaa" "bbbb" "cccc" "dddd" "eeee" "ffff" "gggg" "hhhh" "jjjj"
do
  echo "$NAME" > "$NAME"
  pulp file content upload --relative-path "$NAME" --file "$NAME" || true
  declare $NAME='{"sha256": "'"$(sha256sum --binary $NAME | cut -d" " -f1)"'", "relative_path": "'"$NAME"'"}'
done

pulp file repository content modify --repository test_delete_versions --add-content '['"$aaaa"', '"$bbbb"', '"$cccc"', '"$dddd"', '"$eeee"']'
pulp file repository content modify --repository test_delete_versions --remove-content '['"$bbbb"', '"$cccc"', '"$dddd"', '"$eeee"']' --add-content '['"$ffff"', '"$gggg"', '"$hhhh"', '"$jjjj"']'
pulp file repository content modify --repository test_delete_versions --remove-content '['"$gggg"', '"$jjjj"']' --add-content '['"$cccc"', '"$dddd"']'
pulp file repository content modify --repository test_delete_versions --remove-content '['"$dddd"', '"$hhhh"']' --add-content '['"$eeee"', '"$jjjj"']'

pulp file repository version list --repository test_delete_versions
# pulp file repository content list --repository test_delete_versions

pulp file repository version destroy --repository test_delete_versions --version 2
pulp file repository version list --repository test_delete_versions

if [ ! "$(pulp file repository content list --repository test_delete_versions --version 1 | jq -r '.[].relative_path' | sort)" = $'aaaa\nbbbb\ncccc\ndddd\neeee' ]
then
  echo Version 1 is wrong.
fi
if [ ! "$(pulp file repository content list --repository test_delete_versions --version 3 | jq -r '.[].relative_path' | sort)" = $'aaaa\ncccc\ndddd\nffff\nhhhh' ]
then
  echo Version 3 is wrong.
fi
if [ ! "$(pulp file repository content list --repository test_delete_versions --version 4 | jq -r '.[].relative_path' | sort)" = $'aaaa\ncccc\neeee\nffff\njjjj' ]
then
  echo Version 4 is wrong.
fi
```

**Expected behavior**
Content in the respository versions should not change.

**Additional context**



Discussion:

Resolving patch:
diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py
--- a/pulpcore/app/models/repository.py
+++ b/pulpcore/app/models/repository.py
@@ -4,6 +4,7 @@
 from contextlib import suppress
 from gettext import gettext as _
 from os import path
+from collections import defaultdict
 import logging
 
 import django
@@ -901,9 +902,11 @@ def _squash(self, repo_relations, next_version):
         # delete any relationships added in the version being deleted and removed in the next one.
         repo_relations.filter(version_added=self, version_removed=next_version).delete()
 
-        # If the same content is deleted in version, but added back in next_version
-        # set version_removed field in relation to None, and remove relation adding the content
-        # in next_version
+        # If the same content is deleted in version, but added back in next_version then:
+        # - set version_removed field in relation to version_removed of the relation adding
+        #   the content in next version because the content can be removed again after the
+        #   next_version
+        # - and remove relation adding the content in next_version
         content_added = repo_relations.filter(version_added=next_version).values_list("content_id")
 
         # use list() to force the evaluation of the queryset, otherwise queryset is affected
@@ -914,13 +917,26 @@ def _squash(self, repo_relations, next_version):
             )
         )
 
-        repo_relations.filter(
-            version_removed=self, content_id__in=content_removed_and_readded
-        ).update(version_removed=None)
-
-        repo_relations.filter(
+        repo_contents_readded_in_next_version = repo_relations.filter(
             version_added=next_version, content_id__in=content_removed_and_readded
-        ).delete()
+        )
+
+        # Since the readded contents can be removed again by any subsequent version after the
+        # next version. Get the mapping of readded contents and their versions removed to use
+        # later. The version removed id will be None if a content is not removed.
+        version_removed_id_content_id_map = defaultdict(list)
+        for readded_repo_content in repo_contents_readded_in_next_version.iterator():
+            version_removed_id_content_id_map[readded_repo_content.version_removed_id].append(
+                readded_repo_content.content_id
+            )
+
+        repo_contents_readded_in_next_version.delete()
+
+        # Update the version removed of the readded contents
+        for version_removed_id, content_ids in version_removed_id_content_id_map.items():
+            repo_relations.filter(version_removed=self, content_id__in=content_ids).update(
+                version_removed_id=version_removed_id
+            )
 
         # "squash" by moving other additions and removals forward to the next version
         repo_relations.filter(version_added=self).update(version_added=next_version)


Test patch:
diff --git a/pulpcore/tests/conftest.py b/pulpcore/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/pulpcore/tests/conftest.py
@@ -0,0 +1 @@
+from .conftest_pulp_file import *  # noqa
diff --git a/pulpcore/tests/conftest_pulp_file.py b/pulpcore/tests/conftest_pulp_file.py
new file mode 100644
--- /dev/null
+++ b/pulpcore/tests/conftest_pulp_file.py
@@ -0,0 +1,141 @@
+import logging
+import uuid
+
+from pathlib import Path
+
+import pytest
+
+from pulpcore.client.pulp_file import (
+    ContentFilesApi,
+    RepositoriesFileApi,
+    RepositoriesFileVersionsApi,
+    RemotesFileApi,
+)
+from pulp_smash.pulp3.utils import gen_repo
+
+from pulpcore.tests.functional.api.using_plugin.utils import (
+    gen_file_client,
+)
+
+
+_logger = logging.getLogger(__name__)
+
+
+@pytest.fixture(scope="session")
+def file_client():
+    return gen_file_client()
+
+
+@pytest.fixture(scope="session")
+def content_file_api_client(file_client):
+    return ContentFilesApi(file_client)
+
+
+@pytest.fixture(scope="session")
+def file_repo_api_client(file_client):
+    return RepositoriesFileApi(file_client)
+
+
+@pytest.fixture(scope="session")
+def file_repo_version_api_client(file_client):
+    return RepositoriesFileVersionsApi(file_client)
+
+
+@pytest.fixture
+def file_repo(file_repo_api_client, gen_object_with_cleanup):
+    return gen_object_with_cleanup(file_repo_api_client, gen_repo())
+
+
+@pytest.fixture(scope="session")
+def file_remote_api_client(file_client):
+    return RemotesFileApi(file_client)
+
+
+@pytest.fixture(scope="session")
+def file_fixtures_root():
+    return Path(__file__).parent / "fixtures"
+
+
+@pytest.fixture
+def file_fixture_server_ssl_client_cert_req(
+    ssl_ctx_req_client_auth, file_fixtures_root, gen_fixture_server
+):
+    yield gen_fixture_server(file_fixtures_root, ssl_ctx_req_client_auth)
+
+
+@pytest.fixture
+def file_fixture_server_ssl(ssl_ctx, file_fixtures_root, gen_fixture_server):
+    yield gen_fixture_server(file_fixtures_root, ssl_ctx)
+
+
+@pytest.fixture
+def file_fixture_server(file_fixtures_root, gen_fixture_server):
+    yield gen_fixture_server(file_fixtures_root, None)
+
+
+@pytest.fixture
+def file_fixture_gen_remote(file_fixture_server, file_remote_api_client, gen_object_with_cleanup):
+    def _file_fixture_gen_remote(*, fixture_name, policy, **kwargs):
+        url = file_fixture_server.make_url(f"/{fixture_name}/PULP_MANIFEST")
+        kwargs.update({"url": str(url), "policy": policy, "name": str(uuid.uuid4())})
+        return gen_object_with_cleanup(file_remote_api_client, kwargs)
+
+    yield _file_fixture_gen_remote
+
+
+@pytest.fixture
+def file_fixture_gen_remote_ssl(
+    file_fixture_server_ssl,
+    file_remote_api_client,
+    tls_certificate_authority_cert,
+    gen_object_with_cleanup,
+):
+    def _file_fixture_gen_remote_ssl(*, fixture_name, policy, **kwargs):
+        url = file_fixture_server_ssl.make_url(f"/{fixture_name}/PULP_MANIFEST")
+        kwargs.update(
+            {
+                "url": str(url),
+                "policy": policy,
+                "name": str(uuid.uuid4()),
+                "ca_cert": tls_certificate_authority_cert,
+            }
+        )
+        return gen_object_with_cleanup(file_remote_api_client, kwargs)
+
+    yield _file_fixture_gen_remote_ssl
+
+
+@pytest.fixture
+def file_fixture_gen_remote_client_cert_req(
+    file_fixture_server_ssl_client_cert_req,
+    file_remote_api_client,
+    tls_certificate_authority_cert,
+    client_tls_certificate_cert_pem,
+    client_tls_certificate_key_pem,
+    gen_object_with_cleanup,
+):
+    def _file_fixture_gen_remote_client_cert_req(*, fixture_name, policy, **kwargs):
+        url = file_fixture_server_ssl_client_cert_req.make_url(f"/{fixture_name}/PULP_MANIFEST")
+        kwargs.update(
+            {
+                "url": str(url),
+                "policy": policy,
+                "name": str(uuid.uuid4()),
+                "ca_cert": tls_certificate_authority_cert,
+                "client_cert": client_tls_certificate_cert_pem,
+                "client_key": client_tls_certificate_key_pem,
+            }
+        )
+        return gen_object_with_cleanup(file_remote_api_client, kwargs)
+
+    yield _file_fixture_gen_remote_client_cert_req
+
+
+@pytest.fixture
+def file_fixture_gen_file_repo(file_repo_api_client, gen_object_with_cleanup):
+    """A factory to generate a File Repository with auto-deletion after the test run."""
+
+    def _file_fixture_gen_file_repo(**kwargs):
+        return gen_object_with_cleanup(file_repo_api_client, kwargs)
+
+    yield _file_fixture_gen_file_repo
diff --git a/pulpcore/tests/functional/api/using_plugin/test_repo_versions.py b/pulpcore/tests/functional/api/using_plugin/test_repo_versions.py
--- a/pulpcore/tests/functional/api/using_plugin/test_repo_versions.py
+++ b/pulpcore/tests/functional/api/using_plugin/test_repo_versions.py
@@ -1,8 +1,11 @@
 """Tests related to repository versions."""
 import unittest
+import pytest
 from random import choice, randint, sample
 from time import sleep
 from urllib.parse import urlsplit
+from tempfile import NamedTemporaryFile
+from hashlib import sha256
 
 from pulp_smash import api, config, utils
 from pulp_smash.exceptions import TaskReportError
@@ -396,6 +399,134 @@ def test_delete_publication(self):
             self.client.get(publication["pulp_href"])
 
 
+@pytest.mark.parallel
+def test_squash_repo_version(
+    file_repo_api_client, file_repo_version_api_client, content_file_api_client, file_repo
+):
+    """Test that the deletion of a repository version properly squashes the content.
+
+    - Setup versions like:
+        Version 0: <empty>
+            add: ABCDE
+        Version 1: ABCDE
+            delete: BCDE; add: FGHI
+        Version 2: AFGHI -- to be deleted
+            delete: GI; add: CD
+        Version 3: ACDFH -- to be squashed into
+            delete: DH; add: EI
+        Version 4: ACEFI
+    - Delete version 2.
+    - Check the content of all remaining versions.
+    """
+    content_units = {}
+    for name in ["A", "B", "C", "D", "E", "F", "G", "H", "I"]:
+        try:
+            content_units[name] = content_file_api_client.list(
+                relative_path=name, sha256=sha256(name.encode()).hexdigest()
+            ).results[0]
+        except IndexError:
+            with NamedTemporaryFile() as tf:
+                tf.write(name.encode())
+                tf.flush()
+                response = content_file_api_client.create(relative_path=name, file=tf.name)
+                result = monitor_task(response.task)
+                content_units[name] = content_file_api_client.read(result.created_resources[0])
+    response1 = file_repo_api_client.modify(
+        file_repo.pulp_href,
+        {
+            "add_content_units": [
+                content.pulp_href
+                for key, content in content_units.items()
+                if key in ["A", "B", "C", "D", "E"]
+            ]
+        },
+    )
+
+    response2 = file_repo_api_client.modify(
+        file_repo.pulp_href,
+        {
+            "remove_content_units": [
+                content.pulp_href
+                for key, content in content_units.items()
+                if key in ["B", "C", "D", "E"]
+            ],
+            "add_content_units": [
+                content.pulp_href
+                for key, content in content_units.items()
+                if key in ["F", "G", "H", "I"]
+            ],
+        },
+    )
+
+    response3 = file_repo_api_client.modify(
+        file_repo.pulp_href,
+        {
+            "remove_content_units": [
+                content.pulp_href for key, content in content_units.items() if key in ["G", "I"]
+            ],
+            "add_content_units": [
+                content.pulp_href for key, content in content_units.items() if key in ["C", "D"]
+            ],
+        },
+    )
+
+    response4 = file_repo_api_client.modify(
+        file_repo.pulp_href,
+        {
+            "remove_content_units": [
+                content.pulp_href for key, content in content_units.items() if key in ["D", "H"]
+            ],
+            "add_content_units": [
+                content.pulp_href for key, content in content_units.items() if key in ["E", "I"]
+            ],
+        },
+    )
+    version1 = file_repo_version_api_client.read(monitor_task(response1.task).created_resources[0])
+    version2 = file_repo_version_api_client.read(monitor_task(response2.task).created_resources[0])
+    version3 = file_repo_version_api_client.read(monitor_task(response3.task).created_resources[0])
+    version4 = file_repo_version_api_client.read(monitor_task(response4.task).created_resources[0])
+
+    # Check version state before deletion
+    assert version1.content_summary.added["file.file"]["count"] == 5
+    assert "file.file" not in version1.content_summary.removed
+    assert version2.content_summary.added["file.file"]["count"] == 4
+    assert version2.content_summary.removed["file.file"]["count"] == 4
+    assert version3.content_summary.added["file.file"]["count"] == 2
+    assert version3.content_summary.removed["file.file"]["count"] == 2
+    assert version4.content_summary.added["file.file"]["count"] == 2
+    assert version4.content_summary.removed["file.file"]["count"] == 2
+
+    content1 = content_file_api_client.list(repository_version=version1.pulp_href)
+    content2 = content_file_api_client.list(repository_version=version2.pulp_href)
+    content3 = content_file_api_client.list(repository_version=version3.pulp_href)
+    content4 = content_file_api_client.list(repository_version=version4.pulp_href)
+    assert set((content.relative_path for content in content1.results)) == {"A", "B", "C", "D", "E"}
+    assert set((content.relative_path for content in content2.results)) == {"A", "F", "G", "H", "I"}
+    assert set((content.relative_path for content in content3.results)) == {"A", "C", "D", "F", "H"}
+    assert set((content.relative_path for content in content4.results)) == {"A", "C", "E", "F", "I"}
+
+    monitor_task(file_repo_version_api_client.delete(version2.pulp_href).task)
+
+    # Check version state after deletion (Version 2 is gone...)
+    version1 = file_repo_version_api_client.read(version1.pulp_href)
+    version3 = file_repo_version_api_client.read(version3.pulp_href)
+    version4 = file_repo_version_api_client.read(version4.pulp_href)
+
+    assert version1.content_summary.added["file.file"]["count"] == 5
+    assert "file.file" not in version1.content_summary.removed
+    assert version3.content_summary.added["file.file"]["count"] == 2
+    assert version3.content_summary.removed["file.file"]["count"] == 2
+    assert version4.content_summary.added["file.file"]["count"] == 2
+    assert version4.content_summary.removed["file.file"]["count"] == 2
+
+    content1 = content_file_api_client.list(repository_version=version1.pulp_href)
+    content3 = content_file_api_client.list(repository_version=version3.pulp_href)
+    content4 = content_file_api_client.list(repository_version=version4.pulp_href)
+    assert set((content.relative_path for content in content1.results)) == {"A", "B", "C", "D", "E"}
+    assert set((content.relative_path for content in content3.results)) == {"A", "C", "D", "F", "H"}
+    assert set((content.relative_path for content in content4.results)) == {"A", "C", "E", "F", "I"}
+
+
 class ContentImmutableRepoVersionTestCase(unittest.TestCase):
     """Test whether the content present in a repo version is immutable.