Repository: plone/plone.restapi (language: python)

Issue:
IBlocks SearchableText indexer bypasses plone.textindexer behavior
The `SearchableText_blocks` indexer combines searchable text from blocks with the result of calling the base searchable text extractor, which is hardcoded to be `plone.app.contenttypes.indexers.SearchableText`. However, if the `plone.textindexer` behavior is active, it will be bypassed by this, so `IDynamicTextIndexExtender` adapters won't be used.

Given the way the component architecture handles adapter lookups, there may be no clean solution to call "whatever would have been called for a searchable text indexer if the `IBlocks` one wasn't in place". Another solution would be to turn the logic upside-down and implement the `IBlocks` indexer as an `IDynamicTextIndexExtender` adapter, which would require the `plone.textindexer` behavior to be implied by using Blocks.

I'm not sure what the best or cleanest solution might be. However, Blocks competing with `plone.textindexer` for the actual indexer feels unexpected and possible solutions feel hacky.


Discussion / hints:
This does look a bit tricky. Perhaps we could implement the blocks indexer as an IDynamicTextIndexExtender, like you suggested, and also make IBlocks extend IDexterityTextIndexer, so that enabling blocks implicitly enables the `plone.textindexer` indexer. But, the fact that plone.restapi still aims to be compatible with Plone 5 will make the backwards-compatibility tricky.
See: #1764

Fix patch:
diff --git a/news/1744.bugfix b/news/1744.bugfix
new file mode 100644
index 0000000000..6e261c32d8
--- /dev/null
+++ b/news/1744.bugfix
@@ -0,0 +1,1 @@
+In Plone 6, uses ``plone.textindexer`` to add block texts to the SearchableText index, instead of ``plone.indexer``. This ensures that behaviors can add fields to SearchableText with ``plone.textindexer``. @wesleybl
\ No newline at end of file
diff --git a/src/plone/restapi/behaviors.py b/src/plone/restapi/behaviors.py
index a544f0fbdd..e9389db3d1 100644
--- a/src/plone/restapi/behaviors.py
+++ b/src/plone/restapi/behaviors.py
@@ -2,11 +2,19 @@
 from plone.restapi import _
 from plone.schema import JSONField
 from plone.supermodel import model
+from zope.interface import Interface
 from zope.interface import provider
 
 import json
 
 
+# In Plone 6, IBlocks implicitly enables the IDexterityTextIndexer behavior
+# to handle full-text indexing. In Plone 5, it is not available.
+try:
+    from plone.app.dexterity.textindexer.behavior import IDexterityTextIndexer
+except ImportError:
+    IDexterityTextIndexer = Interface
+
 BLOCKS_SCHEMA = json.dumps({"type": "object", "properties": {}})
 
 LAYOUT_SCHEMA = json.dumps(
@@ -18,7 +26,7 @@
 
 
 @provider(IFormFieldProvider)
-class IBlocks(model.Schema):
+class IBlocks(model.Schema, IDexterityTextIndexer):
 
     model.fieldset("layout", label=_("Layout"), fields=["blocks", "blocks_layout"])
 
diff --git a/src/plone/restapi/indexers.py b/src/plone/restapi/indexers.py
index cfb65587e5..5362192463 100644
--- a/src/plone/restapi/indexers.py
+++ b/src/plone/restapi/indexers.py
@@ -1,12 +1,6 @@
-# XXX: EXPERIMENTAL!!!
-# This is an experimental feature meant for use in Volto only!
-# This code is likely to change in the future, even within minor releases.
-# We will make sure plone.restapi latest always works with the latest Volto release.
-# This code is planned to being refactored into plone.volto before CMFPlone 6.0 is out.
-# <tisto@plone.org>
-
 from plone.app.contenttypes.indexers import SearchableText
 from plone.indexer.decorator import indexer
+from plone.restapi import HAS_PLONE_6
 from plone.restapi.behaviors import IBlocks
 from plone.restapi.blocks import visit_subblocks
 from plone.restapi.interfaces import IBlockSearchableText
@@ -88,7 +82,6 @@ def extract_text(block, obj, request):
     :param request: Current request.
     :returns: A string with text found in the block.
     """
-    result = ""
     block_type = block.get("@type", "")
     # searchableText is the conventional way of storing
     # searchable info in a block
@@ -107,8 +100,7 @@ def extract_text(block, obj, request):
     return result
 
 
-@indexer(IBlocks)
-def SearchableText_blocks(obj):
+def get_blocks_text(obj):
     """Extract text to be used by the SearchableText index in the Catalog."""
     request = getRequest()
     blocks = obj.blocks
@@ -117,8 +109,35 @@ def SearchableText_blocks(obj):
     for block_id in blocks_layout.get("items", []):
         block = blocks.get(block_id, {})
         blocks_text.append(extract_text(block, obj, request))
-
-    # Extract text using the base plone.app.contenttypes indexer
-    std_text = SearchableText(obj)
-    blocks_text.append(std_text)
-    return " ".join([text.strip() for text in blocks_text if text.strip()])
+    return blocks_text
+
+
+def text_strip(text_list):
+    return " ".join([text.strip() for text in text_list if text.strip()])
+
+
+if HAS_PLONE_6:
+    # In Plone 6, uses IDynamicTextIndexExtender to index block texts.
+    # This ensures that indexing with plone.textindexer continues to work. See:
+    # https://github.com/plone/plone.restapi/issues/1744
+    from plone.app.dexterity import textindexer
+
+    @implementer(textindexer.IDynamicTextIndexExtender)
+    @adapter(IBlocks)
+    class BlocksSearchableTextExtender(object):
+        def __init__(self, context):
+            self.context = context
+
+        def __call__(self):
+            return text_strip(get_blocks_text(self.context))
+
+else:
+    # BBB: Plone 5.2 does not have plone.app.dexterity.textindexer.
+    # So we need to index with plone.indexer.
+    @indexer(IBlocks)
+    def SearchableText_blocks(obj):
+        blocks_text = get_blocks_text(obj)
+        # Extract text using the base plone.app.contenttypes indexer
+        std_text = SearchableText(obj)
+        blocks_text.append(std_text)
+        return text_strip(blocks_text)
diff --git a/src/plone/restapi/indexers.zcml b/src/plone/restapi/indexers.zcml
index 345a065ecb..3d5c78ffdd 100644
--- a/src/plone/restapi/indexers.zcml
+++ b/src/plone/restapi/indexers.zcml
@@ -4,23 +4,27 @@
     i18n_domain="plone"
     >
 
-  <configure zcml:condition="installed plone.app.contenttypes">
-    <adapter
-        factory=".indexers.SearchableText_blocks"
-        name="SearchableText"
-        />
-    <adapter
-        factory=".indexers.TextBlockSearchableText"
-        name="text"
-        />
-    <adapter
-        factory=".indexers.SlateTextIndexer"
-        name="slate"
-        />
-    <adapter
-        factory=".indexers.TableBlockSearchableText"
-        name="table"
-        />
-  </configure>
+  <adapter
+      factory=".indexers.BlocksSearchableTextExtender"
+      name="IBlocks"
+      zcml:condition="have plone-60"
+      />
+  <adapter
+      factory=".indexers.SearchableText_blocks"
+      name="SearchableText"
+      zcml:condition="not-have plone-60"
+      />
+  <adapter
+      factory=".indexers.TextBlockSearchableText"
+      name="text"
+      />
+  <adapter
+      factory=".indexers.SlateTextIndexer"
+      name="slate"
+      />
+  <adapter
+      factory=".indexers.TableBlockSearchableText"
+      name="table"
+      />
 
 </configure>


Test patch:
diff --git a/src/plone/restapi/tests/test_blocks_searchable_text.py b/src/plone/restapi/tests/test_blocks_searchable_text.py
index e979dce99c..5f82129fbe 100644
--- a/src/plone/restapi/tests/test_blocks_searchable_text.py
+++ b/src/plone/restapi/tests/test_blocks_searchable_text.py
@@ -7,7 +7,7 @@
 from plone.dexterity.utils import createContentInContainer
 from plone.restapi.behaviors import IBlocks
 from plone.restapi.interfaces import IBlockSearchableText
-from plone.restapi.testing import PLONE_RESTAPI_DX_FUNCTIONAL_TESTING
+from plone.restapi.testing import PLONE_RESTAPI_BLOCKS_FUNCTIONAL_TESTING
 from plone.restapi.testing import RelativeSession
 from zope.component import adapter
 from zope.component import provideAdapter
@@ -21,7 +21,7 @@
 
 class TestSearchTextInBlocks(unittest.TestCase):
 
-    layer = PLONE_RESTAPI_DX_FUNCTIONAL_TESTING
+    layer = PLONE_RESTAPI_BLOCKS_FUNCTIONAL_TESTING
 
     def setUp(self):
         self.app = self.layer["app"]
diff --git a/src/plone/restapi/tests/test_indexers.py b/src/plone/restapi/tests/test_indexers.py
index 0e227a4c9f..671a6836bf 100644
--- a/src/plone/restapi/tests/test_indexers.py
+++ b/src/plone/restapi/tests/test_indexers.py
@@ -1,6 +1,6 @@
 from plone.dexterity.interfaces import IDexterityFTI
-from plone.restapi.indexers import SearchableText_blocks
-from plone.restapi.testing import PLONE_RESTAPI_DX_INTEGRATION_TESTING
+from plone.restapi.testing import PLONE_RESTAPI_BLOCKS_FUNCTIONAL_TESTING
+from Products.CMFCore.utils import getToolByName
 from uuid import uuid4
 from zope.component import queryUtility
 
@@ -89,11 +89,12 @@
 
 class TestSearchableTextIndexer(unittest.TestCase):
 
-    layer = PLONE_RESTAPI_DX_INTEGRATION_TESTING
+    layer = PLONE_RESTAPI_BLOCKS_FUNCTIONAL_TESTING
 
     def setUp(self):
         self.portal = self.layer["portal"]
         self.request = self.layer["request"]
+        self.catalog = getToolByName(self.portal, "portal_catalog")
         fti = queryUtility(IDexterityFTI, name="Document")
         behavior_list = [a for a in fti.behaviors]
         behavior_list.append("volto.blocks")
@@ -108,10 +109,12 @@ def setUp(self):
         )
         self.document = self.portal["doc1"]
 
-    @staticmethod
-    def _extract_searchable_text(obj):
-        indexer = SearchableText_blocks(obj)
-        return indexer()
+    def _extract_searchable_text(self, obj):
+        query = {"path": {"query": "/".join(obj.getPhysicalPath()), "depth": 0}}
+        brains = self.catalog(query)
+        brain = brains[0]
+        data = self.catalog.getIndexDataForRID(brain.getRID())
+        return " ".join(data["SearchableText"])
 
     @staticmethod
     def _add_blocks(obj, raw_blocks):
@@ -119,11 +122,12 @@ def _add_blocks(obj, raw_blocks):
         layout = list(blocks.keys())
         obj.blocks = blocks
         obj.blocks_layout["items"] = layout
+        obj.reindexObject()
 
     def test_indexer_no_blocks(self):
         result = self._extract_searchable_text(self.document)
-        self.assertIn("Title is here", result)
-        self.assertIn("Description is there", result)
+        self.assertIn("title is here", result)
+        self.assertIn("description is there", result)
 
     def test_indexer_block_has_searchableText(self):
         document = self.document
@@ -132,8 +136,8 @@ def test_indexer_block_has_searchableText(self):
             [{"@type": "new-block", "attribute": "bar", "searchableText": "Foo Bar"}],
         )
         result = self._extract_searchable_text(document)
-        self.assertIn("Title is here Description is there", result)
-        self.assertIn("Foo Bar", result)
+        self.assertIn("title is here description is there", result)
+        self.assertIn("foo bar", result)
 
     def test_indexer_multiple_blocks(self):
         document = self.document
@@ -145,7 +149,7 @@ def test_indexer_multiple_blocks(self):
             ],
         )
         result = self._extract_searchable_text(document)
-        self.assertIn("Plone is a CMS ", result)
+        self.assertIn("plone is a cms", result)
 
     def test_indexer_block_slate_bbb(self):
         document = self.document
@@ -156,8 +160,8 @@ def test_indexer_block_slate_bbb(self):
             ],
         )
         result = self._extract_searchable_text(document)
-        self.assertIn("Title is here Description is there", result)
-        self.assertIn("Follow Plone Conference", result)
+        self.assertIn("title is here description is there", result)
+        self.assertIn("follow plone conference", result)
 
     def test_indexer_block_text(self):
         document = self.document
@@ -168,8 +172,8 @@ def test_indexer_block_text(self):
             ],
         )
         result = self._extract_searchable_text(document)
-        self.assertIn("Title is here Description is there", result)
-        self.assertIn("Plone is a powerful content management system", result)
+        self.assertIn("title is here description is there", result)
+        self.assertIn("plone is a powerful content management system", result)
 
     def test_indexer_block_table(self):
         document = self.document
@@ -180,8 +184,8 @@ def test_indexer_block_table(self):
             ],
         )
         result = self._extract_searchable_text(document)
-        self.assertIn("Title is here Description is there", result)
-        self.assertIn("My data", result)
+        self.assertIn("title is here description is there", result)
+        self.assertIn("my data", result)
 
     def test_indexer_block_with_subblocks(self):
         document = self.document
@@ -208,8 +212,8 @@ def test_indexer_block_with_subblocks(self):
         )
         result = self._extract_searchable_text(document)
         # From Slate sub block
-        self.assertIn("Follow Plone Conference", result)
+        self.assertIn("follow plone conference", result)
         # From DraftJS
-        self.assertIn("Plone is a powerful content management system", result)
+        self.assertIn("plone is a powerful content management system", result)
         # From Table block
-        self.assertIn("My data", result)
+        self.assertIn("my data", result)