Repository: pypi/warehouse

Issue:
Search relevancy is still not ideal
#1020 fixed #1019 for the majority of packages, however a few still produce odd results:

For example:
https://warehouse.python.org/search/?q=flask (`Flask` package is 2nd, `Flask-Admin` is first)
https://warehouse.python.org/search/?q=django (`Django` package is 11th, `dotulu` is first)
https://warehouse.python.org/search/?q=git (First 3 packages do not have "git" anywhere in them)

This is hard to test in dev because the dev DB is a snapshot of TestPyPI, and those packages are missing.

@dstufft, would it be possible to get a more complete DB for local development?



Discussion:
Yea, I'll see if I can't just dump the regular PyPI database and sanitize it.

Sorry for the delay. I have a snaphot of the real PyPI I can send you, but I'd rather not add it to the repository because even compressed it's quite large (~270MB) and although I sanitized it I'd rather not risk having missed something. Is there an email address or something I can send you a link via?

Assuming you're still interested in this anyways!

Sure! You can use the address on my github profile.


Resolving patch:
diff --git a/warehouse/packaging/search.py b/warehouse/packaging/search.py
--- a/warehouse/packaging/search.py
+++ b/warehouse/packaging/search.py
@@ -21,12 +21,18 @@
     filter=["standard", "lowercase", "stop", "snowball"],
 )
 
+NameAnalyzer = analyzer(
+    "normalized_name",
+    tokenizer="lowercase",
+    filter=["standard", "lowercase", "word_delimiter"],
+)
+
 
 @doc_type
 class Project(DocType):
 
     name = String()
-    normalized_name = String(index="not_analyzed")
+    normalized_name = String(analyzer=NameAnalyzer, index_options="docs")
     version = String(index="not_analyzed", multi=True)
     summary = String(analyzer="snowball")
     description = String(analyzer="snowball")
diff --git a/warehouse/views.py b/warehouse/views.py
--- a/warehouse/views.py
+++ b/warehouse/views.py
@@ -157,9 +157,10 @@ def search(request):
             "multi_match",
             query=request.params["q"],
             fields=[
-                "name^2", "version", "author", "author_email", "maintainer",
-                "maintainer_email", "home_page", "license", "summary",
-                "description", "keywords", "platform", "download_url",
+                "author", "author_email", "description^5", "download_url",
+                "home_page", "keywords^5", "license", "maintainer",
+                "maintainer_email", "normalized_name^10", "platform",
+                "summary^5",
             ],
         ).suggest(
             name="name_suggestion",


Test patch:
diff --git a/tests/unit/test_views.py b/tests/unit/test_views.py
--- a/tests/unit/test_views.py
+++ b/tests/unit/test_views.py
@@ -141,10 +141,10 @@ def test_with_a_query(self, monkeypatch, db_request, page):
                 "multi_match",
                 query="foo bar",
                 fields=[
-                    "name^2", "version", "author", "author_email",
-                    "maintainer", "maintainer_email", "home_page", "license",
-                    "summary", "description", "keywords", "platform",
-                    "download_url",
+                    "author", "author_email", "description^5", "download_url",
+                    "home_page", "keywords^5", "license", "maintainer",
+                    "maintainer_email", "normalized_name^10", "platform",
+                    "summary^5",
                 ],
             ),
         ]
@@ -198,10 +198,10 @@ def test_with_an_ordering(self, monkeypatch, db_request, page):
                 "multi_match",
                 query="foo bar",
                 fields=[
-                    "name^2", "version", "author", "author_email",
-                    "maintainer", "maintainer_email", "home_page", "license",
-                    "summary", "description", "keywords", "platform",
-                    "download_url",
+                    "author", "author_email", "description^5", "download_url",
+                    "home_page", "keywords^5", "license", "maintainer",
+                    "maintainer_email", "normalized_name^10", "platform",
+                    "summary^5",
                 ],
             ),
         ]
@@ -270,10 +270,10 @@ def test_with_classifiers(self, monkeypatch, db_request, page):
                 "multi_match",
                 query="foo bar",
                 fields=[
-                    "name^2", "version", "author", "author_email",
-                    "maintainer", "maintainer_email", "home_page", "license",
-                    "summary", "description", "keywords", "platform",
-                    "download_url",
+                    "author", "author_email", "description^5", "download_url",
+                    "home_page", "keywords^5", "license", "maintainer",
+                    "maintainer_email", "normalized_name^10", "platform",
+                    "summary^5",
                 ],
             ),
         ]