Repository: internetarchive/openlibrary

Issue:
Make solr search alternative_title field
Feature. This field is comprised of all the edition titles, but does not include the work title. This results in better matches when searching for books without their original language in mind. We should include the work title in here during the next re-index.


### Technical
<!-- What should be noted about the implementation? -->

### Testing

- Try it out on testing.openlibrary.org with "All" selected in the drop down. Do a Before/After on https://codepen.io/cdrini/full/wvJqzaK 


### Screenshot

https://docs.google.com/spreadsheets/d/1BN5I7-OkTPaoTr2Es6jQ4O9ICWFmH0q9CP6kEgolCgg/edit#gid=1006480604

Much better rankings for our search set. This means that the works we want to appear high for user queries are now appearing higher.

Search Class | AVERAGE of Prod Ranking | AVERAGE of Testing Ranking
-- | -- | --
Author | 31.40952381 | 25.82698413
Non-English Books | 68 | 34.33333333
Popular Books | 10.97142857 | 8.228571429
Series | 57.98518519 | 31.84126984
**Grand Total** | **25.19967716** | **16.73720743**


Search Class | AVERAGE of Prod Ranking | AVERAGE of Testing Ranking
-- | -- | --
Drini | 36.2952381 | 25.16190476
Top user searches | 40.1447619 | 32.02190476
Wikipedia most published | 15.97222222 | 7.543650794
**Grand Total** | **25.19967716** | **16.73720743**


### Stakeholders
<!-- @ tag stakeholders of this bug -->


<!-- Attribution Disclaimer: By proposing this pull request, I affirm to have made a best-effort and exercised my discretion to make sure relevant sections of this code which substantially leverage code suggestions, code generation, or code snippets from sources (e.g. Stack Overflow, GitHub) have been annotated with basic attribution so reviewers & contributors may have confidence and access to the correct context to evaluate and use this code. -->



Discussion:
Would love some help testing this from the community! CC @seabelis @dcapillae @LeadSongDog @mheiman 

This is a small change but will have a big impact on the "All" search results. Try it out on https://codepen.io/cdrini/full/wvJqzaK for comparing openlibrary.org to testing.openlibrary.org . The one on the right is the "after".

Resolving patch:
diff --git a/openlibrary/config.py b/openlibrary/config.py
--- a/openlibrary/config.py
+++ b/openlibrary/config.py
@@ -1,6 +1,7 @@
 """Utility for loading config file.
 """
 import os
+import sys
 import yaml
 import infogami
 from infogami import config
@@ -17,6 +18,9 @@ def load(config_file):
 
     WARNING: This function is deprecated, please use load_config instead.
     """
+    if 'pytest' in sys.modules:
+        # During pytest ensure we're not using like olsystem or something
+        assert config_file == 'conf/openlibrary.yml'
     # for historic reasons
     global runtime_config
     with open(config_file) as in_file:
@@ -28,6 +32,9 @@ def load_config(config_file):
 
     The loaded config will be available via infogami.config.
     """
+    if 'pytest' in sys.modules:
+        # During pytest ensure we're not using like olsystem or something
+        assert config_file == 'conf/openlibrary.yml'
     infogami.load_config(config_file)
     setup_infobase_config(config_file)
 
diff --git a/openlibrary/plugins/worksearch/code.py b/openlibrary/plugins/worksearch/code.py
--- a/openlibrary/plugins/worksearch/code.py
+++ b/openlibrary/plugins/worksearch/code.py
@@ -23,6 +23,7 @@
 from openlibrary.plugins.inside.code import fulltext_search
 from openlibrary.plugins.openlibrary.processors import urlsafe
 from openlibrary.plugins.upstream.utils import urlencode
+from openlibrary.solr.update_work import get_solr_next
 from openlibrary.utils import escape_bracket
 from openlibrary.utils.ddc import (
     normalize_ddc,
@@ -38,14 +39,6 @@
 
 logger = logging.getLogger("openlibrary.worksearch")
 
-if hasattr(config, 'plugin_worksearch'):
-    solr_select_url = (
-        config.plugin_worksearch.get('solr_base_url', 'localhost') + '/select'
-    )
-
-    default_spellcheck_count = config.plugin_worksearch.get('spellcheck_count', 10)
-
-
 ALL_FIELDS = [
     "key",
     "redirects",
@@ -111,6 +104,10 @@
     'editions': 'edition_count',
     'by': 'author_name',
     'publishers': 'publisher',
+    'subtitle': 'alternative_subtitle',
+    **({'title': 'alternative_title'} if get_solr_next() else {}),
+    'work_subtitle': 'subtitle',
+    'work_title': 'title',
     # "Private" fields
     # This is private because we'll change it to a multi-valued field instead of a
     # plain string at the next opportunity, which will make it much more usable.
@@ -173,6 +170,13 @@
 
 plurals = {f + 's': f for f in ('publisher', 'author')}
 
+if hasattr(config, 'plugin_worksearch'):
+    solr_select_url = (
+        config.plugin_worksearch.get('solr_base_url', 'localhost') + '/select'
+    )
+
+    default_spellcheck_count = config.plugin_worksearch.get('spellcheck_count', 10)
+
 
 @public
 def get_solr_works(work_key: Iterable[str]) -> dict[str, dict]:
@@ -496,7 +500,10 @@ def run_solr_query(
         if use_dismax:
             params.append(('q', ' '.join(q_list)))
             params.append(('defType', 'dismax'))
-            params.append(('qf', 'text title^20 author_name^20'))
+            if get_solr_next():
+                params.append(('qf', 'text alternative_title^20 author_name^20'))
+            else:
+                params.append(('qf', 'text title^20 author_name^20'))
             params.append(('bf', 'min(100,edition_count)'))
         else:
             params.append(('q', ' '.join(q_list + ['_val_:"sqrt(edition_count)"^10'])))
diff --git a/openlibrary/solr/update_work.py b/openlibrary/solr/update_work.py
--- a/openlibrary/solr/update_work.py
+++ b/openlibrary/solr/update_work.py
@@ -7,7 +7,7 @@
 from json import JSONDecodeError
 from math import ceil
 from statistics import median
-from typing import Literal, List, Optional, cast, TypedDict, Any, Union
+from typing import Iterable, Literal, List, Optional, cast, TypedDict, Any, Union
 
 import httpx
 import requests
@@ -578,8 +578,8 @@ def add_list(name, values):
         add('title', w.get('title'))
         add('subtitle', w.get('subtitle'))
 
-        add_list("alternative_title", self.get_alternate_titles(w, editions))
-        add_list('alternative_subtitle', self.get_alternate_subtitles(w, editions))
+        add_list("alternative_title", self.get_alternate_titles((w, *editions)))
+        add_list('alternative_subtitle', self.get_alternate_subtitles((w, *editions)))
 
         add('edition_count', len(editions))
 
@@ -657,40 +657,27 @@ def get_edition_ddcs(ed: dict):
 
         return d
 
-    def get_alternate_titles(self, w, editions):
-        """
-        Get titles from the editions as alternative titles.
-
-        :param dict w:
-        :param list[dict] editions:
-        :rtype: set[str]
-        """
+    @staticmethod
+    def get_alternate_titles(books: Iterable[dict]) -> set[str]:
+        """Get titles from the editions as alternative titles."""
         result = set()
-        for e in editions:
-            result.add(e.get('title'))
-            result.update(e.get('work_titles', []))
-            result.update(e.get('other_titles', []))
-
-        # Remove original title and None.
-        # None would've got in if any of the editions has no title.
-        result.discard(None)
-        result.discard(w.get('title'))
-        return result
+        for bookish in books:
+            full_title = bookish.get('title')
+            if not full_title:
+                # None would've got in if any of the editions has no title.
+                continue
+            if bookish.get('subtitle'):
+                full_title += ': ' + bookish['subtitle']
+            result.add(full_title)
+            result.update(bookish.get('work_titles', []))
+            result.update(bookish.get('other_titles', []))
 
-    def get_alternate_subtitles(self, w, editions):
-        """
-        Get subtitles from the editions as alternative titles.
+        return result
 
-        :param dict w:
-        :param list[dict] editions:
-        :rtype: set[str]
-        """
-        subtitle = w.get('subtitle')
-        return {
-            e['subtitle']
-            for e in editions
-            if e.get('subtitle') and e['subtitle'] != subtitle
-        }
+    @staticmethod
+    def get_alternate_subtitles(books: Iterable[dict]) -> set[str]:
+        """Get subtitles from the editions as alternative titles."""
+        return {bookish['subtitle'] for bookish in books if bookish.get('subtitle')}
 
     def get_isbns(self, editions):
         """


Test patch:
diff --git a/openlibrary/plugins/worksearch/tests/test_worksearch.py b/openlibrary/plugins/worksearch/tests/test_worksearch.py
--- a/openlibrary/plugins/worksearch/tests/test_worksearch.py
+++ b/openlibrary/plugins/worksearch/tests/test_worksearch.py
@@ -65,30 +65,30 @@ def test_sorted_work_editions():
 QUERY_PARSER_TESTS = {
     'No fields': ('query here', [{'field': 'text', 'value': 'query here'}]),
     'Author field': (
-        'title:food rules author:pollan',
+        'food rules author:pollan',
         [
-            {'field': 'title', 'value': 'food rules'},
+            {'field': 'text', 'value': 'food rules'},
             {'field': 'author_name', 'value': 'pollan'},
         ],
     ),
     'Field aliases': (
         'title:food rules by:pollan',
         [
-            {'field': 'title', 'value': 'food rules'},
+            {'field': 'alternative_title', 'value': 'food rules'},
             {'field': 'author_name', 'value': 'pollan'},
         ],
     ),
     'Fields are case-insensitive aliases': (
-        'title:food rules By:pollan',
+        'food rules By:pollan',
         [
-            {'field': 'title', 'value': 'food rules'},
+            {'field': 'text', 'value': 'food rules'},
             {'field': 'author_name', 'value': 'pollan'},
         ],
     ),
     'Quotes': (
         'title:"food rules" author:pollan',
         [
-            {'field': 'title', 'value': '"food rules"'},
+            {'field': 'alternative_title', 'value': '"food rules"'},
             {'field': 'author_name', 'value': 'pollan'},
         ],
     ),
@@ -96,7 +96,7 @@ def test_sorted_work_editions():
         'query here title:food rules author:pollan',
         [
             {'field': 'text', 'value': 'query here'},
-            {'field': 'title', 'value': 'food rules'},
+            {'field': 'alternative_title', 'value': 'food rules'},
             {'field': 'author_name', 'value': 'pollan'},
         ],
     ),
@@ -109,7 +109,10 @@ def test_sorted_work_editions():
     'Colons in field': (
         'title:flatland:a romance of many dimensions',
         [
-            {'field': 'title', 'value': r'flatland\:a romance of many dimensions'},
+            {
+                'field': 'alternative_title',
+                'value': r'flatland\:a romance of many dimensions',
+            },
         ],
     ),
     'Operators': (
@@ -229,7 +232,7 @@ def test_build_q_list():
     }
     expect = (
         [
-            'title:((Holidays are Hell))',
+            'alternative_title:((Holidays are Hell))',
             'author_name:((Kim Harrison))',
             'OR',
             'author_name:((Lynsay Sands))',
@@ -237,7 +240,7 @@ def test_build_q_list():
         False,
     )
     query_fields = [
-        {'field': 'title', 'value': '(Holidays are Hell)'},
+        {'field': 'alternative_title', 'value': '(Holidays are Hell)'},
         {'field': 'author_name', 'value': '(Kim Harrison)'},
         {'op': 'OR'},
         {'field': 'author_name', 'value': '(Lynsay Sands)'},
diff --git a/openlibrary/tests/core/test_fulltext.py b/openlibrary/tests/core/test_fulltext.py
--- a/openlibrary/tests/core/test_fulltext.py
+++ b/openlibrary/tests/core/test_fulltext.py
@@ -6,8 +6,8 @@
 
 
 class Test_fulltext_search_api:
-    def test_config(self):
-        assert not hasattr(config, "plugin_inside")
+    def test_no_config(self, monkeypatch):
+        monkeypatch.delattr(config, "plugin_inside")
         response = fulltext.fulltext_search_api({})
         assert response == {"error": "Unable to prepare search engine"}
 
diff --git a/openlibrary/tests/solr/test_update_work.py b/openlibrary/tests/solr/test_update_work.py
--- a/openlibrary/tests/solr/test_update_work.py
+++ b/openlibrary/tests/solr/test_update_work.py
@@ -347,6 +347,18 @@ async def test_with_one_printdisabled_edition(self):
         assert d['edition_count'] == 1
         assert d['ebook_count_i'] == 1
 
+    def test_get_alternate_titles(self):
+        f = SolrProcessor.get_alternate_titles
+
+        no_title = {}
+        only_title = {'title': 'foo'}
+        with_subtitle = {'title': 'foo 2', 'subtitle': 'bar'}
+
+        assert f([]) == set()
+        assert f([no_title]) == set()
+        assert f([only_title, no_title]) == {'foo'}
+        assert f([with_subtitle, only_title]) == {'foo 2: bar', 'foo'}
+
     @pytest.mark.asyncio
     async def test_with_multiple_editions(self):
         w = make_work()