Repository: conan-io/conan
Pull request #16605 (2024-07-03 22:07:50)

Refactor/search
Changelog: Omit
Docs: Omit

Refactor, continue moving things to ``conan`` scope and cleaning things



Patch:
diff --git a/conan/api/subapi/cache.py b/conan/api/subapi/cache.py
index e8504efc1..19a6db3da 100644
--- a/conan/api/subapi/cache.py
+++ b/conan/api/subapi/cache.py
@@ -9,7 +9,7 @@ from conan.api.output import ConanOutput
 from conan.internal.cache.cache import PkgCache
 from conan.internal.cache.home_paths import HomePaths
 from conan.internal.conan_app import ConanApp
-from conan.internal.integrity_check import IntegrityChecker
+from conan.internal.cache.integrity_check import IntegrityChecker
 from conans.client.downloaders.download_cache import DownloadCache
 from conans.errors import ConanException
 from conans.model.package_ref import PkgReference
diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py
index a7452ecfe..fecf61ab3 100644
--- a/conan/api/subapi/list.py
+++ b/conan/api/subapi/list.py
@@ -1,14 +1,18 @@
+import os
+from collections import OrderedDict
 from typing import Dict
 
 from conan.api.model import PackagesList
 from conan.api.output import ConanOutput, TimedOutput
+from conan.internal.api.list.query_parse import filter_package_configs
 from conan.internal.conan_app import ConanApp
+from conan.internal.paths import CONANINFO
 from conans.errors import ConanException, NotFoundException
 from conans.model.info import load_binary_info
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference, ref_matches
-from conans.search.search import get_cache_packages_binary_info, filter_packages
 from conans.util.dates import timelimit
+from conans.util.files import load
 
 
 class ListAPI:
@@ -69,7 +73,7 @@ class ListAPI:
         if not remote:
             app = ConanApp(self.conan_api)
             prefs = app.cache.get_package_references(ref)
-            packages = get_cache_packages_binary_info(app.cache, prefs)
+            packages = _get_cache_packages_binary_info(app.cache, prefs)
         else:
             app = ConanApp(self.conan_api)
             if ref.revision == "latest":
@@ -85,7 +89,18 @@ class ListAPI:
         :param query: str like "os=Windows AND (arch=x86 OR compiler=gcc)"
         :return: Dict[PkgReference, PkgConfiguration]
         """
-        return filter_packages(query, pkg_configurations)
+        if query is None:
+            return pkg_configurations
+        try:
+            if "!" in query:
+                raise ConanException("'!' character is not allowed")
+            if "~" in query:
+                raise ConanException("'~' character is not allowed")
+            if " not " in query or query.startswith("not "):
+                raise ConanException("'not' operator is not allowed")
+            return filter_package_configs(pkg_configurations, query)
+        except Exception as exc:
+            raise ConanException("Invalid package query: %s. %s" % (query, exc))
 
     @staticmethod
     def filter_packages_profile(packages, profile, ref):
@@ -361,3 +376,29 @@ class _BinaryDistance:
                 "python_requires": self.python_requires_diff,
                 "confs": self.confs_diff,
                 "explanation": self.explanation()}
+
+
+def _get_cache_packages_binary_info(cache, prefs) -> Dict[PkgReference, dict]:
+    """
+    param package_layout: Layout for the given reference
+    """
+
+    result = OrderedDict()
+
+    for pref in prefs:
+        latest_prev = cache.get_latest_package_reference(pref)
+        pkg_layout = cache.pkg_layout(latest_prev)
+
+        # Read conaninfo
+        info_path = os.path.join(pkg_layout.package(), CONANINFO)
+        if not os.path.exists(info_path):
+            raise ConanException(f"Corrupted package '{pkg_layout.reference}' "
+                                 f"without conaninfo.txt in: {info_path}")
+        conan_info_content = load(info_path)
+        info = load_binary_info(conan_info_content)
+        pref = pkg_layout.reference
+        # The key shoudln't have the latest package revision, we are asking for package configs
+        pref.revision = None
+        result[pkg_layout.reference] = info
+
+    return result
diff --git a/conan/api/subapi/search.py b/conan/api/subapi/search.py
index b70f1aa02..582b753c6 100644
--- a/conan/api/subapi/search.py
+++ b/conan/api/subapi/search.py
@@ -1,5 +1,4 @@
 from conan.internal.conan_app import ConanApp
-from conans.search.search import search_recipes
 
 
 class SearchAPI:
@@ -17,7 +16,7 @@ class SearchAPI:
         if remote:
             refs = app.remote_manager.search_recipes(remote, query)
         else:
-            references = search_recipes(app.cache, query)
+            references = app.cache.search_recipes(query)
             # For consistency with the remote search, we return references without revisions
             # user could use further the API to look for the revisions
             refs = []
diff --git a/conans/search/__init__.py b/conan/internal/api/list/__init__.py
similarity index 100%
rename from conans/search/__init__.py
rename to conan/internal/api/list/__init__.py
diff --git a/conans/search/query_parse.py b/conan/internal/api/list/query_parse.py
similarity index 65%
rename from conans/search/query_parse.py
rename to conan/internal/api/list/query_parse.py
index 2cc717ab0..8e33e078d 100644
--- a/conans/search/query_parse.py
+++ b/conan/internal/api/list/query_parse.py
@@ -1,14 +1,59 @@
+from collections import OrderedDict
 
-def is_operator(el):
+
+def filter_package_configs(pkg_configurations, query):
+    postfix = _infix_to_postfix(query) if query else []
+    result = OrderedDict()
+    for pref, data in pkg_configurations.items():
+        if _evaluate_postfix_with_info(postfix, data):
+            result[pref] = data
+    return result
+
+
+def _evaluate_postfix_with_info(postfix, binary_info):
+
+    # Evaluate conaninfo with the expression
+
+    def evaluate_info(expression):
+        """Receives an expression like compiler.version="12"
+        Uses conan_vars_info in the closure to evaluate it"""
+        name, value = expression.split("=", 1)
+        value = value.replace("\"", "")
+        return _evaluate(name, value, binary_info)
+
+    return _evaluate_postfix(postfix, evaluate_info)
+
+
+def _evaluate(prop_name, prop_value, binary_info):
+    """
+    Evaluates a single prop_name, prop_value like "os", "Windows" against
+    conan_vars_info.serialize_min()
+    """
+
+    def compatible_prop(setting_value, _prop_value):
+        return (_prop_value == setting_value) or (_prop_value == "None" and setting_value is None)
+
+    # TODO: Necessary to generalize this query evaluation to include all possible fields
+    info_settings = binary_info.get("settings", {})
+    info_options = binary_info.get("options", {})
+
+    if not prop_name.startswith("options."):
+        return compatible_prop(info_settings.get(prop_name), prop_value)
+    else:
+        prop_name = prop_name[len("options."):]
+        return compatible_prop(info_options.get(prop_name), prop_value)
+
+
+def _is_operator(el):
     return el in ["|", "&"]
 
 
 def _parse_expression(subexp):
-    '''Expressions like:
+    """Expressions like:
      compiler.version=12
      compiler="Visual Studio"
      arch="x86"
-     Could be replaced with another one to parse different queries'''
+     Could be replaced with another one to parse different queries """
     ret = ""
     quoted = False
     for char in subexp:
@@ -19,7 +64,7 @@ def _parse_expression(subexp):
 
         if quoted:
             ret += char
-        elif char == " " or is_operator(char) or char in [")", "("]:
+        elif char == " " or _is_operator(char) or char in [")", "("]:
             break
         else:
             ret += char
@@ -30,7 +75,7 @@ def _parse_expression(subexp):
     return ret
 
 
-def evaluate_postfix(postfix, evaluator):
+def _evaluate_postfix(postfix, evaluator):
     """
     Evaluates a postfix expression and returns a boolean
     @param postfix:  Postfix expression as a list
@@ -43,7 +88,7 @@ def evaluate_postfix(postfix, evaluator):
 
     stack = []
     for el in postfix:
-        if not is_operator(el):
+        if not _is_operator(el):
             stack.append(el)
         else:
             o1 = stack.pop()
@@ -66,7 +111,7 @@ def evaluate_postfix(postfix, evaluator):
         return stack[0]
 
 
-def infix_to_postfix(exp):
+def _infix_to_postfix(exp):
     """
     Translates an infix expression to postfix using an standard algorithm
     with little hacks for parse complex expressions like "compiler.version=4"
@@ -98,11 +143,11 @@ def infix_to_postfix(exp):
                     output.append(popped)
             if popped != "(":
                 raise Exception("Bad expression, not balanced parenthesis")
-        elif is_operator(char):
+        elif _is_operator(char):
             # Same operations has the same priority
             # replace this lines if the operators need to have
             # some priority
-            if stack and is_operator(stack[:-1]):
+            if stack and _is_operator(stack[:-1]):
                 popped = stack.pop()
                 output.append(popped)
             stack.append(char)
diff --git a/conan/internal/cache/cache.py b/conan/internal/cache/cache.py
index 92af0a603..5a8e754f2 100644
--- a/conan/internal/cache/cache.py
+++ b/conan/internal/cache/cache.py
@@ -1,7 +1,9 @@
 import hashlib
 import os
+import re
 import shutil
 import uuid
+from fnmatch import translate
 from typing import List
 
 from conan.internal.cache.conan_reference_layout import RecipeLayout, PackageLayout
@@ -168,8 +170,18 @@ class PkgCache:
         assert ref.timestamp
         self._db.update_recipe_timestamp(ref)
 
-    def all_refs(self):
-        return self._db.list_references()
+    def search_recipes(self, pattern=None, ignorecase=True):
+        # Conan references in main storage
+        if pattern:
+            if isinstance(pattern, RecipeReference):
+                pattern = repr(pattern)
+            pattern = translate(pattern)
+            pattern = re.compile(pattern, re.IGNORECASE if ignorecase else 0)
+
+        refs = self._db.list_references()
+        if pattern:
+            refs = [r for r in refs if r.partial_match(pattern)]
+        return refs
 
     def exists_prev(self, pref):
         # Used just by download to skip downloads if prev already exists in cache
diff --git a/conan/internal/integrity_check.py b/conan/internal/cache/integrity_check.py
similarity index 100%
rename from conan/internal/integrity_check.py
rename to conan/internal/cache/integrity_check.py
diff --git a/conans/client/graph/range_resolver.py b/conans/client/graph/range_resolver.py
index 535cc5ce2..31df11c37 100644
--- a/conans/client/graph/range_resolver.py
+++ b/conans/client/graph/range_resolver.py
@@ -2,7 +2,6 @@ from conans.client.graph.proxy import should_update_reference
 from conans.errors import ConanException
 from conans.model.recipe_ref import RecipeReference
 from conans.model.version_range import VersionRange
-from conans.search.search import search_recipes
 
 
 class RangeResolver:
@@ -57,7 +56,7 @@ class RangeResolver:
         local_found = self._cached_cache.get(pattern)
         if local_found is None:
             # This local_found is weird, it contains multiple revisions, not just latest
-            local_found = search_recipes(self._cache, pattern)
+            local_found = self._cache.search_recipes(pattern)
             # TODO: This is still necessary to filter user/channel, until search_recipes is fixed
             local_found = [ref for ref in local_found if ref.user == search_ref.user
                            and ref.channel == search_ref.channel]
diff --git a/conans/model/recipe_ref.py b/conans/model/recipe_ref.py
index 0d86ff3fa..ca58c5984 100644
--- a/conans/model/recipe_ref.py
+++ b/conans/model/recipe_ref.py
@@ -187,6 +187,23 @@ class RecipeReference:
             return not condition
         return condition
 
+    def partial_match(self, pattern):
+        """
+        Finds if pattern matches any of partial sums of tokens of conan reference
+        """
+        tokens = [self.name, "/", str(self.version)]
+        if self.user:
+            tokens += ["@", self.user]
+        if self.channel:
+            tokens += ["/", self.channel]
+        if self.revision:
+            tokens += ["#", self.revision]
+        partial = ""
+        for token in tokens:
+            partial += token
+            if pattern.match(partial):
+                return True
+
 
 def ref_matches(ref, pattern, is_consumer):
     if not ref or not str(ref):
diff --git a/conans/search/search.py b/conans/search/search.py
deleted file mode 100644
index c0adea1e7..000000000
--- a/conans/search/search.py
+++ /dev/null
@@ -1,130 +0,0 @@
-import os
-import re
-from collections import OrderedDict
-from fnmatch import translate
-from typing import Dict
-
-from conans.errors import ConanException
-from conans.model.info import load_binary_info
-from conans.model.package_ref import PkgReference
-from conans.model.recipe_ref import RecipeReference
-from conan.internal.paths import CONANINFO
-from conans.search.query_parse import evaluate_postfix, infix_to_postfix
-from conans.util.files import load
-
-
-def filter_packages(query, results: Dict[PkgReference, dict]):
-    if query is None:
-        return results
-    try:
-        if "!" in query:
-            raise ConanException("'!' character is not allowed")
-        if "~" in query:
-            raise ConanException("'~' character is not allowed")
-        if " not " in query or query.startswith("not "):
-            raise ConanException("'not' operator is not allowed")
-        postfix = infix_to_postfix(query) if query else []
-        result = OrderedDict()
-        for pref, data in results.items():
-            if _evaluate_postfix_with_info(postfix, data):
-                result[pref] = data
-        return result
-    except Exception as exc:
-        raise ConanException("Invalid package query: %s. %s" % (query, exc))
-
-
-def _evaluate_postfix_with_info(postfix, binary_info):
-
-    # Evaluate conaninfo with the expression
-
-    def evaluate_info(expression):
-        """Receives an expression like compiler.version="12"
-        Uses conan_vars_info in the closure to evaluate it"""
-        name, value = expression.split("=", 1)
-        value = value.replace("\"", "")
-        return _evaluate(name, value, binary_info)
-
-    return evaluate_postfix(postfix, evaluate_info)
-
-
-def _evaluate(prop_name, prop_value, binary_info):
-    """
-    Evaluates a single prop_name, prop_value like "os", "Windows" against
-    conan_vars_info.serialize_min()
-    """
-
-    def compatible_prop(setting_value, _prop_value):
-        return (_prop_value == setting_value) or (_prop_value == "None" and setting_value is None)
-
-    # TODO: Necessary to generalize this query evaluation to include all possible fields
-    info_settings = binary_info.get("settings", {})
-    info_options = binary_info.get("options", {})
-
-    if not prop_name.startswith("options."):
-        return compatible_prop(info_settings.get(prop_name), prop_value)
-    else:
-        prop_name = prop_name[len("options."):]
-        return compatible_prop(info_options.get(prop_name), prop_value)
-
-
-def search_recipes(cache, pattern=None, ignorecase=True):
-    # Conan references in main storage
-    if pattern:
-        if isinstance(pattern, RecipeReference):
-            pattern = repr(pattern)
-        pattern = translate(pattern)
-        pattern = re.compile(pattern, re.IGNORECASE) if ignorecase else re.compile(pattern)
-
-    refs = cache.all_refs()
-    if pattern:
-        _refs = []
-        for r in refs:
-            match_ref = str(r) if not r.revision else repr(r)
-            if _partial_match(pattern, match_ref):
-                _refs.append(r)
-        refs = _refs
-    return refs
-
-
-def _partial_match(pattern, reference):
-    """
-    Finds if pattern matches any of partial sums of tokens of conan reference
-    """
-    tokens = reference.replace('/', ' / ').replace('@', ' @ ').replace('#', ' # ').split()
-
-    def partial_sums(iterable):
-        partial = ''
-        for i in iterable:
-            partial += i
-            yield partial
-
-    return any(map(pattern.match, list(partial_sums(tokens))))
-
-
-def get_cache_packages_binary_info(cache, prefs) -> Dict[PkgReference, dict]:
-    """
-    param package_layout: Layout for the given reference
-    """
-
-    result = OrderedDict()
-
-    package_layouts = []
-    for pref in prefs:
-        latest_prev = cache.get_latest_package_reference(pref)
-        package_layouts.append(cache.pkg_layout(latest_prev))
-
-    for pkg_layout in package_layouts:
-        # Read conaninfo
-        info_path = os.path.join(pkg_layout.package(), CONANINFO)
-        if not os.path.exists(info_path):
-            raise ConanException(f"Corrupted package '{pkg_layout.reference}' "
-                                 f"without conaninfo.txt in: {info_path}")
-        conan_info_content = load(info_path)
-
-        info = load_binary_info(conan_info_content)
-        pref = pkg_layout.reference
-        # The key shoudln't have the latest package revision, we are asking for package configs
-        pref.revision = None
-        result[pkg_layout.reference] = info
-
-    return result
diff --git a/conans/server/service/v2/search.py b/conans/server/service/v2/search.py
index 66ec092ff..b91c4f6e2 100644
--- a/conans/server/service/v2/search.py
+++ b/conans/server/service/v2/search.py
@@ -7,7 +7,6 @@ from conans.errors import ForbiddenException, RecipeNotFoundException
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
 from conan.internal.paths import CONANINFO
-from conans.search.search import _partial_match
 from conans.server.utils.files import list_folder_subdirs
 from conans.util.files import load
 
@@ -93,7 +92,7 @@ class SearchService(object):
                 fields_dir = [underscore_to_none(d) for d in subdir.split("/")]
                 new_ref = RecipeReference(*fields_dir)
                 new_ref.revision = None
-                if _partial_match(b_pattern, repr(new_ref)):
+                if new_ref.partial_match(b_pattern):
                     ret.add(new_ref)
 
             return sorted(ret)


Test patch:
diff --git a/.ci/jenkins/testsv2.jenkins b/.ci/jenkins/testsv2.jenkins
index 3e0085ea4..08ba87fff 100644
--- a/.ci/jenkins/testsv2.jenkins
+++ b/.ci/jenkins/testsv2.jenkins
@@ -79,7 +79,7 @@ private Closure runTests(String nodeName, String pythonVersion, String module) {
                                      + "conan --version && conan --help && "
 
 
-                def launchTests = "python -m pytest ${module} -n=4 --durations=20"
+                def launchTests = "python -m pytest ${module} -n=4 --durations=20 -rs"
 
                 if (shouldTestServer(module)) {
                     def rm = (nodeName=="Windows") ? "del" : "rm"
diff --git a/test/unittests/search/search_query_parse_test.py b/test/unittests/search/search_query_parse_test.py
index 2504d9eca..c3177d4a6 100644
--- a/test/unittests/search/search_query_parse_test.py
+++ b/test/unittests/search/search_query_parse_test.py
@@ -1,28 +1,28 @@
 import unittest
 
-from conans.search.query_parse import evaluate_postfix, infix_to_postfix
+from conan.internal.api.list.query_parse import _evaluate_postfix, _infix_to_postfix
 
 
 class QueryParseTest(unittest.TestCase):
 
     def test_get_postfix(self):
-        r = infix_to_postfix("")
+        r = _infix_to_postfix("")
         self.assertEqual(r, [])
 
-        r = infix_to_postfix("a=2")
+        r = _infix_to_postfix("a=2")
         self.assertEqual(r, ["a=2"])
 
-        r = infix_to_postfix("a=2 OR b=3")
+        r = _infix_to_postfix("a=2 OR b=3")
         self.assertEqual(r, ["a=2", "b=3", "|"])
 
-        r = infix_to_postfix("a= OR b=")
+        r = _infix_to_postfix("a= OR b=")
         self.assertEqual(r, ["a=", "b=", "|"])  # Equivalent to ""
 
-        r = infix_to_postfix("(a=2 OR b=3) AND (j=34 AND j=45) OR (a=1)")
+        r = _infix_to_postfix("(a=2 OR b=3) AND (j=34 AND j=45) OR (a=1)")
         self.assertEqual(r, ["a=2", "b=3", "|", "j=34", "j=45", "&", "a=1", "&", "|"])
 
         with self.assertRaisesRegex(Exception, "Invalid expression: 2"):
-            r = infix_to_postfix("a= 2 OR b=3")
+            r = _infix_to_postfix("a= 2 OR b=3")
 
     def test_evaluate_postfix(self):
 
@@ -30,8 +30,8 @@ class QueryParseTest(unittest.TestCase):
             return expr in ("a=2", "j=45")
 
         def evaluate(q):
-            r = infix_to_postfix(q)
-            return evaluate_postfix(r, evaluator)
+            r = _infix_to_postfix(q)
+            return _evaluate_postfix(r, evaluator)
 
         self.assertTrue(evaluate("a=2"))
         self.assertFalse(evaluate("a=4"))