Repository: plone/Products.CMFPlone

Issue:
Fix broken relations
## BUG/PROBLEM REPORT (OR OTHER COMMON ISSUE)

Content with broken relations should be repaired by `Products.relationhelper.rebuild_relations` (flush of intIds or no flush of intIds)

### What I did:

Create page 1
Create page 2
Make page 2 related content of page 1.
Delete page 2.
Execute fix of relations (flush or no flush of intIds) via control panel relations. This calls `Products.relationhelper.rebuild_relations`.

### What I expect to happen:

The broken relation is removed by `Products.relationhelper.rebuild_relations`. 

### What actually happened:

The broken relation is NOT removed by `Products.relationhelper.rebuild_relations`. 

### What version of Plone/ Addons I am using:

Plone 6.0.3



Discussion:

Resolving patch:
diff --git a/Products/CMFPlone/relationhelper.py b/Products/CMFPlone/relationhelper.py
--- a/Products/CMFPlone/relationhelper.py
+++ b/Products/CMFPlone/relationhelper.py
@@ -1,5 +1,6 @@
 from collections import Counter
 from collections import defaultdict
+from collections import OrderedDict
 from five.intid.intid import addIntIdSubscriber
 from plone.app.linkintegrity.handlers import modifiedContent
 from plone.app.linkintegrity.utils import referencedRelationship
@@ -51,6 +52,7 @@ def rebuild_relations(context=None, flush_and_rebuild_intids=False):
     else:
         cleanup_intids()
     restore_relations()
+    log_relations()
 
 
 def get_relations_stats():
@@ -62,7 +64,7 @@ def get_relations_stats():
             rel = relation_catalog.resolveRelationToken(token)
         except ObjectMissingError:
             broken["Object is missing"] += 1
-            logger.info(f"Token {token} has no object.")
+            logger.warning(f"Token {token} has no object.")
             continue
 
         if rel.isBroken():
@@ -73,8 +75,13 @@ def get_relations_stats():
 
 
 def get_all_relations():
-    """Get all data from zc.relation catalog.
-    Logs some useful statistics.
+    """Get relations from zc.relation catalog.
+
+    Log statistics.
+    Return list of dictionaries:
+        from_uuid: source UID if available else None
+        to_uuid: target UID if available else None
+        from_attribute: relation name
     """
     results = []
     info = defaultdict(int)
@@ -84,25 +91,27 @@ def get_all_relations():
         try:
             rel = relation_catalog.resolveRelationToken(token)
         except ObjectMissingError:
-            logger.info(f"Token {token} has no object.")
+            logger.warning(f"No relation for relation token '{token}'.")
             continue
 
-        if rel.from_object and rel.to_object:
-            try:
-                results.append(
-                    {
-                        "from_uuid": rel.from_object.UID(),
-                        "to_uuid": rel.to_object.UID(),
-                        "from_attribute": rel.from_attribute,
-                    }
-                )
-                info[rel.from_attribute] += 1
-            except AttributeError as ex:
-                logger.info(f"Something went wrong while storing {rel}: \n {ex}")
-        else:
-            logger.info(
-                f"Dropping relation {rel.from_attribute} from {rel.from_object} to {rel.to_object}"
+        rel_uid_info = OrderedDict()
+        rel_uid_info["from_attribute"] = rel.from_attribute
+        try:
+            rel_uid_info["from_uuid"] = (
+                rel.from_object.UID() if rel.from_object else None
             )
+        except AttributeError as e:
+            rel_uid_info["from_uuid"] = None
+            logger.warning(f"Broken relation: {rel_uid_info} '{str(e)}'")
+        try:
+            rel_uid_info["to_uuid"] = rel.to_object.UID() if rel.to_object else None
+        except AttributeError as e:
+            rel_uid_info["to_uuid"] = None
+            logger.warning(f"Broken relation: {rel_uid_info} '{str(e)}'")
+        info[rel.from_attribute] += 1
+        results.append(rel_uid_info)
+
+    # Log stats
     msg = ""
     for key, value in info.items():
         msg += f"{key}: {value}\n"
@@ -115,131 +124,142 @@ def store_relations(context=None):
     all_relations = get_all_relations()
     portal = getSite()
     IAnnotations(portal)[RELATIONS_KEY] = all_relations
-    logger.info(f"Stored {len(all_relations)} relations on the portal")
 
 
 def purge_relations(context=None):
     """Removes all entries form zc.relation catalog.
+
     RelationValues that were set as attribute on content are still there!
     These are removed/overwritten when restoring the relations.
     """
-    rel_catalog = getUtility(ICatalog)
-    rel_catalog.clear()
-    logger.info("Purged zc.relation catalog")
+    relation_catalog = getUtility(ICatalog)
+    relation_catalog.clear()
+    logger.info("zc.relation catalog purged.")
 
 
 def restore_relations(context=None, all_relations=None):
-    """Restore relations from a annotation on the portal."""
+    """Restore relations from an annotation on the portal."""
 
     portal = getSite()
     if all_relations is None:
         all_relations = IAnnotations(portal)[RELATIONS_KEY]
-    logger.info(f"Loaded {len(all_relations)} relations to restore")
+    logger.info(f"Loaded {len(all_relations)} relations to restore.")
     update_linkintegrity = set()
     modified_items = set()
     modified_relation_lists = defaultdict(list)
 
-    # remove duplicates but keep original order
+    # Remove duplicates but keep original order.
     unique_relations = []
     seen = set()
-    seen_add = seen.add
     for rel in all_relations:
         hashable = tuple(rel.items())
         if hashable not in seen:
             unique_relations.append(rel)
-            seen_add(hashable)
+            seen.add(hashable)
         else:
             logger.info(f"Dropping duplicate: {hashable}")
 
     if len(unique_relations) < len(all_relations):
         logger.info(f"Dropping {len(all_relations) - len(unique_relations)} duplicates")
-        all_relations = unique_relations
 
+    # Update relations.
     intids = getUtility(IIntIds)
-    for index, item in enumerate(all_relations, start=1):
-        if not index % 500:
-            logger.info(f"Restored {index} of {len(all_relations)} relations...")
-
-        try:
-            source_obj = uuidToObject(item["from_uuid"])
-        except KeyError:
-            # brain exists but no object
-            source_obj = None
-        try:
-            target_obj = uuidToObject(item["to_uuid"])
-        except KeyError:
-            # brain exists but no object
-            target_obj = None
-
-        if not source_obj:
-            logger.info(f'{item["from_uuid"]} is missing')
-            continue
-
-        if not target_obj:
-            logger.info(f'{item["to_uuid"]} is missing')
+    new_index = 0
+    for index, item in enumerate(unique_relations, start=1):
+        # Get source object for UID. Skip relation if no object found.
+        if item["from_uuid"] is None:
+            logger.warning(f"No source object. {tuple(item.items())}.")
             continue
+        else:
+            try:
+                source_obj = uuidToObject(item["from_uuid"])
+            except KeyError:
+                # brain exists but no object
+                source_obj = None
 
+        # Get target object of UID. Do not skip relation, but update source_obj below.
+        if item["to_uuid"] is None:
+            target_obj = None
+        else:
+            try:
+                target_obj = uuidToObject(item["to_uuid"])
+            except KeyError:
+                # brain exists but no object
+                target_obj = None
+        if target_obj is None:
+            logger.warning(f"No target object. {tuple(item.items())}")
+            # The source_obj will be updated to remove the broken relation below.
+
+        # source_obj and target_obj are dexterity content types.
         if not IDexterityContent.providedBy(source_obj):
-            logger.info(f"{source_obj} is no dexterity content")
+            logger.warning(f"Source {source_obj} is no dexterity content.")
             continue
-
         if not IDexterityContent.providedBy(target_obj):
-            logger.info(f"{target_obj} is no dexterity content")
-            continue
+            logger.warning(f"Target {target_obj} is no dexterity content.")
 
-        from_attribute = item["from_attribute"]
+        # Get intId for target_obj.
         try:
             to_id = intids.getId(target_obj)
         except KeyError:
-            logger.warning(f"No intid for {target_obj}")
-            continue
+            logger.warning(f"No intId for {target_obj}")
+            to_id = None
 
+        # Postpone linkintegrity check.
+        from_attribute = item["from_attribute"]
         if from_attribute == referencedRelationship:
-            # Ignore linkintegrity for now. We'll rebuilt it at the end!
             update_linkintegrity.add(item["from_uuid"])
             continue
 
+        # Working copy relations
         if HAS_ITERATE and from_attribute == ITERATE_RELATION_NAME:
             # Iterate relations are not set as values of fields
-            relation = StagingRelationValue(to_id)
-            event._setRelation(source_obj, ITERATE_RELATION_NAME, relation)
+            if to_id:
+                relation = StagingRelationValue(to_id)
+                event._setRelation(source_obj, ITERATE_RELATION_NAME, relation)
             continue
 
+        # Relations not based on schema field
         field_and_schema = get_field_and_schema_for_fieldname(
             from_attribute, source_obj.portal_type
         )
         if field_and_schema is None:
             # the from_attribute is no field
-            logger.info(f"No field. Setting relation: {item}")
+            logger.info(f"No field. Setting relation: {tuple(item.items())}")
             event._setRelation(source_obj, from_attribute, RelationValue(to_id))
             continue
 
         field, schema = field_and_schema
-        relation = RelationValue(to_id)
+        relationvalue = RelationValue(to_id) if to_id else None
 
+        # schema field relations: RelationList, RelationChoice, Relation
+        #
+        # RelationList
         if isinstance(field, RelationList):
-            logger.info(
-                f"Add relation to relationslist {from_attribute} from {source_obj.absolute_url()} to {target_obj.absolute_url()}"
-            )
+            if not target_obj:
+                logger.warning(
+                    f"Broken relation not restored: {from_attribute} from {source_obj.absolute_url()}"
+                )
+
             if item["from_uuid"] in modified_relation_lists.get(from_attribute, []):
                 # Do not purge relations
                 existing_relations = getattr(source_obj, from_attribute, [])
             else:
                 # First touch. Make sure we purge!
                 existing_relations = []
-            existing_relations.append(relation)
+            if relationvalue:
+                existing_relations.append(relationvalue)
             setattr(source_obj, from_attribute, existing_relations)
             modified_items.add(item["from_uuid"])
             modified_relation_lists[from_attribute].append(item["from_uuid"])
-            continue
 
+        # Relation, RelationChoice
         elif isinstance(field, (Relation, RelationChoice)):
-            logger.info(
-                f"Add relation {from_attribute} from {source_obj.absolute_url()} to {target_obj.absolute_url()}"
-            )
-            setattr(source_obj, from_attribute, relation)
+            if not target_obj:
+                logger.info(
+                    f"Broken relation not restored: {from_attribute} from {source_obj.absolute_url()}"
+                )
+            setattr(source_obj, from_attribute, relationvalue)
             modified_items.add(item["from_uuid"])
-            continue
 
         else:
             # we should never end up here!
@@ -247,11 +267,18 @@ def restore_relations(context=None, all_relations=None):
                 f"Unexpected relation {from_attribute} from {source_obj.absolute_url()} to {target_obj.absolute_url()}"
             )
 
+        new_index += 1
+        if not new_index % 500:
+            logger.info(f"Restored {new_index} relations.")
+
+    # Link integrity
     update_linkintegrity = set(update_linkintegrity)
-    logger.info(f"Updating linkintegrity for {len(update_linkintegrity)} items")
+    logger.info(f"Updating linkintegrity for {len(update_linkintegrity)} items.")
     for uuid in sorted(update_linkintegrity):
         modifiedContent(uuidToObject(uuid), None)
-    logger.info(f"Updating relations for {len(modified_items)} items")
+
+    # Reindex relations in relations catalog.
+    logger.info(f"Updating relations for {len(modified_items)} items.")
     for uuid in sorted(modified_items):
         obj = uuidToObject(uuid)
         # updateRelations from z3c.relationfield does not properly update relations in behaviors
@@ -261,10 +288,25 @@ def restore_relations(context=None, all_relations=None):
         updateRelations(obj, None)
         update_behavior_relations(obj, None)
 
-    # purge annotation from portal if they exist
+    # Purge annotation from portal.
     if RELATIONS_KEY in IAnnotations(portal):
         del IAnnotations(portal)[RELATIONS_KEY]
-    logger.info("Done!")
+
+    logger.info("All valid relations restored.")
+
+
+def log_relations():
+    info, broken = get_relations_stats()
+    msg = ""
+    for key, value in info.items():
+        msg += f"{key}: {value}\n"
+    logger.info(f"\nRestored relations: \n{msg}")
+
+    if len(broken.items()) > 0:
+        msg = ""
+        for key, value in broken.items():
+            msg += f"{key}: {value}\n"
+        logger.info(f"\nStill broken relations: \n{msg}")
 
 
 def get_intid(obj):
@@ -291,6 +333,7 @@ def get_field_and_schema_for_fieldname(field_id, portal_type):
 
 
 def cleanup_intids(context=None):
+    logger.info("Clean up intIds.")
     intids = getUtility(IIntIds)
     all_refs = [
         f"{i.object.__class__.__module__}.{i.object.__class__.__name__}"
@@ -298,6 +341,7 @@ def cleanup_intids(context=None):
     ]
     logger.info(Counter(all_refs))
 
+    # Unregister RelationValues
     count = 0
     refs = [i for i in intids.refs.values() if isinstance(i.object, RelationValue)]
     for ref in refs:
@@ -305,31 +349,33 @@ def cleanup_intids(context=None):
         count += 1
     logger.info(f"Removed all {count} RelationValues from IntId-tool")
 
+    # Unregister broken references
     count = 0
     for ref in intids.refs.values():
         if "broken" in repr(ref.object):
             intids.unregister(ref)
-    logger.info(f"Removed {count} broken refs from IntId-tool")
+    logger.info(f"Removed {count} broken references from IntId-tool")
+
     all_refs = [
-        "{i.object.__class__.__module__}.{i.object.__class__.__name__}"
+        f"{i.object.__class__.__module__}.{i.object.__class__.__name__}"
         for i in intids.refs.values()
     ]
     logger.info(Counter(all_refs))
 
 
 def flush_intids():
-    """Flush all intids"""
+    """Flush all intIds"""
     intids = getUtility(IIntIds)
     intids.ids = intids.family.OI.BTree()
     intids.refs = intids.family.IO.BTree()
 
 
 def rebuild_intids():
-    """Create new intids"""
+    """Create new intIds"""
 
     def add_to_intids(obj, path):
         if IContentish.providedBy(obj):
-            logger.info(f"Added {obj} at {path} to intid")
+            # logger.info(f"Added intId for {obj} at {path} to intId utility.")
             addIntIdSubscriber(obj, None)
 
     portal = getSite()


Test patch:
diff --git a/Products/CMFPlone/tests/test_relationhelper.py b/Products/CMFPlone/tests/test_relationhelper.py
new file mode 100644
--- /dev/null
+++ b/Products/CMFPlone/tests/test_relationhelper.py
@@ -0,0 +1,139 @@
+from plone import api
+from plone.app.testing import setRoles
+from plone.app.testing import TEST_USER_ID
+from Products.CMFPlone.testing import PRODUCTS_CMFPLONE_INTEGRATION_TESTING
+from zope.component import getUtility
+from Products.CMFPlone.relationhelper import (
+    get_relations_stats,
+    purge_relations,
+    rebuild_relations,
+    restore_relations,
+    store_relations,
+)
+from zope.intid.interfaces import IIntIds
+
+import unittest
+
+
+class TestRelationhelper(unittest.TestCase):
+    layer = PRODUCTS_CMFPLONE_INTEGRATION_TESTING
+
+    def setUp(self):
+        self.portal = self.layer["portal"]
+        self.request = self.layer["request"]
+        setRoles(self.portal, TEST_USER_ID, ("Manager",))
+
+    def test_relations_stats(self):
+        doc1 = api.content.create(type="Document", title="doc1", container=self.portal)
+        doc2 = api.content.create(type="Document", title="doc2", container=self.portal)
+        api.relation.create(doc1, doc2, "relatedItems")
+
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {"relatedItems": 1})
+        self.assertEqual(dict(broken), {})
+
+    def test_relations_stats_broken(self):
+        doc1 = api.content.create(type="Document", title="doc1", container=self.portal)
+        doc2 = api.content.create(type="Document", title="doc2", container=self.portal)
+        api.relation.create(doc1, doc2, "relatedItems")
+        self.portal._delObject("doc2")
+
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {})
+        self.assertEqual(dict(broken), {"relatedItems": 1})
+
+    def test_rebuild_relations(self):
+        doc1 = api.content.create(type="Document", title="doc1", container=self.portal)
+        doc2 = api.content.create(type="Document", title="doc2", container=self.portal)
+        doc3 = api.content.create(type="Document", title="doc3", container=self.portal)
+        intids = getUtility(IIntIds)
+        doc1_intid = intids.getId(doc1)
+        doc2_intid = intids.getId(doc2)
+        doc3_intid = intids.getId(doc3)
+
+        api.relation.create(doc1, doc2, "relatedItems")
+        api.relation.create(doc1, doc3, "relatedItems")
+
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {"relatedItems": 2})
+        self.assertEqual(dict(broken), {})
+
+        rebuild_relations()
+
+        # Relations are the same after a rebuild.
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {"relatedItems": 2})
+        self.assertEqual(dict(broken), {})
+
+        # intids are not changed.
+        doc1_intid_after = intids.getId(doc1)
+        doc2_intid_after = intids.getId(doc2)
+        doc3_intid_after = intids.getId(doc3)
+        self.assertEqual(doc1_intid, doc1_intid_after)
+        self.assertEqual(doc2_intid, doc2_intid_after)
+        self.assertEqual(doc3_intid, doc3_intid_after)
+
+        # Break a relation by target.
+        self.portal._delObject("doc2")
+
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {"relatedItems": 1})
+        self.assertEqual(dict(broken), {"relatedItems": 1})
+
+        # Broken relations are removed after rebuilding.
+        rebuild_relations()
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {"relatedItems": 1})
+        self.assertEqual(dict(broken), {})
+
+        # Break a relation by source.
+        self.portal._delObject("doc1")
+
+        # Broken relation is removed. No rebuild necessary.
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {})
+        self.assertEqual(dict(broken), {})
+
+    def test_rebuild_relations_with_intid_flush(self):
+        doc1 = api.content.create(type="Document", title="doc1", container=self.portal)
+        doc2 = api.content.create(type="Document", title="doc2", container=self.portal)
+        doc3 = api.content.create(type="Document", title="doc3", container=self.portal)
+        intids = getUtility(IIntIds)
+        doc1_intid = intids.getId(doc1)
+        doc2_intid = intids.getId(doc2)
+        doc3_intid = intids.getId(doc3)
+
+        api.relation.create(doc1, doc2, "relatedItems")
+        api.relation.create(doc1, doc3, "relatedItems")
+
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {"relatedItems": 2})
+        self.assertEqual(dict(broken), {})
+
+        rebuild_relations(flush_and_rebuild_intids=True)
+
+        # Relations are the same after a rebuild.
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {"relatedItems": 2})
+        self.assertEqual(dict(broken), {})
+
+        # intids are now changed.
+        doc1_intid_after = intids.getId(doc1)
+        doc2_intid_after = intids.getId(doc2)
+        doc3_intid_after = intids.getId(doc3)
+        self.assertNotEqual(doc1_intid, doc1_intid_after)
+        self.assertNotEqual(doc2_intid, doc2_intid_after)
+        self.assertNotEqual(doc3_intid, doc3_intid_after)
+
+        # Break a relation.
+        self.portal._delObject("doc2")
+
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {"relatedItems": 1})
+        self.assertEqual(dict(broken), {"relatedItems": 1})
+
+        # Broken relations are gone after rebuilding.
+        rebuild_relations(flush_and_rebuild_intids=True)
+        stats, broken = get_relations_stats()
+        self.assertEqual(dict(stats), {"relatedItems": 1})
+        self.assertEqual(dict(broken), {})