Repository: mabel-dev/opteryx

Issue:
🧹 lemmatize full text search



Discussion:

Resolving patch:
diff --git a/opteryx/__version__.py b/opteryx/__version__.py
--- a/opteryx/__version__.py
+++ b/opteryx/__version__.py
@@ -1,4 +1,4 @@
-__build__ = 424
+__build__ = 426
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/opteryx/functions/__init__.py b/opteryx/functions/__init__.py
--- a/opteryx/functions/__init__.py
+++ b/opteryx/functions/__init__.py
@@ -342,8 +342,11 @@ def select_values(boolean_arrays, value_arrays):
     "GET": _get,
     "GET_STRING": _get_string,
     "LIST_CONTAINS": _iterate_double_parameter(other_functions.list_contains),
+    "ARRAY_CONTAINS": _iterate_double_parameter(other_functions.list_contains),
     "LIST_CONTAINS_ANY": other_functions.list_contains_any,
+    "ARRAY_CONTAINS_ANY": other_functions.list_contains_any,
     "LIST_CONTAINS_ALL": _iterate_double_parameter(other_functions.list_contains_all),
+    "ARRAY_CONTAINS_ALL": _iterate_double_parameter(other_functions.list_contains_all),
     "SEARCH": other_functions.search,
     "COALESCE": _coalesce,
     "IFNULL": other_functions.if_null,
diff --git a/opteryx/functions/other_functions.py b/opteryx/functions/other_functions.py
--- a/opteryx/functions/other_functions.py
+++ b/opteryx/functions/other_functions.py
@@ -149,29 +149,30 @@ def cosine_similarity(arr, val):
     """
     ad hoc cosine similarity function, slow.
     """
-    import re
-    import string
 
     from opteryx.compiled.functions import tokenize_and_remove_punctuation
     from opteryx.compiled.functions import vectorize
     from opteryx.virtual_datasets.stop_words import STOP_WORDS
 
-    # Compile a regular expression pattern that matches any punctuation
-    punctuation_pattern = re.compile(r"[{}]".format(re.escape(string.punctuation)))
-
     def cosine_similarity(
         vec1: numpy.ndarray, vec2: numpy.ndarray, vec2_norm: numpy.float32
     ) -> float:
         vec1 = vec1.astype(numpy.float32)
-        return numpy.dot(vec1, vec2) / (numpy.linalg.norm(vec1) * vec2_norm)
+        vec1_norm = numpy.linalg.norm(vec1)
+
+        product = vec1_norm * vec2_norm
+        if product == 0:
+            return 0
+
+        return numpy.dot(vec1, vec2) / product
 
     # import time
 
     if len(val) == 0:
         return []
-    tokenized_literal = list(tokenize_and_remove_punctuation(str(val[0]), STOP_WORDS))
+    tokenized_literal = tokenize_and_remove_punctuation(str(val[0]), STOP_WORDS)
     if len(tokenized_literal) == 0:
-        return []
+        return [0.0] * len(arr)
     # print(len(val))
 
     # t = time.monotonic_ns()
@@ -185,6 +186,9 @@ def cosine_similarity(
     comparison_vector = vectors[-1].astype(numpy.float32)
     comparison_vector_norm = numpy.linalg.norm(comparison_vector)
 
+    if comparison_vector_norm == 0.0:
+        return [0.0] * len(val)
+
     # t = time.monotonic_ns()
     similarities = [
         cosine_similarity(vector, comparison_vector, comparison_vector_norm)


Test patch:
diff --git a/tests/misc/test_documentation.py b/tests/misc/test_documentation.py
--- a/tests/misc/test_documentation.py
+++ b/tests/misc/test_documentation.py
@@ -226,6 +226,35 @@ def get_user_permissions(user_roles):
     assert perms == {"Query", "Execute", "Analyze"}
 
 
+def test_membership_permissions():
+    import opteryx
+
+    conn = opteryx.connect(memberships=["Apollo 11"])
+    curr = conn.cursor()
+
+    # the missions field is an ARRAY
+    curr.execute("SELECT * FROM $astronauts WHERE ARRAY_CONTAINS_ANY(missions, @@user_memberships)")
+    assert curr.rowcount == 3
+
+    res = opteryx.query(
+        "SELECT * FROM $astronauts WHERE ARRAY_CONTAINS_ANY(missions, @@user_memberships)",
+        memberships=["Apollo 11"],
+    )
+    assert res.rowcount == 3
+
+    curr = conn.cursor()
+    curr.execute(
+        "SELECT $missions.* FROM $missions INNER JOIN $user ON Mission = value WHERE attribute = 'membership'"
+    )
+    assert curr.rowcount == 1
+
+    res = opteryx.query(
+        "SELECT $missions.* FROM $missions INNER JOIN $user ON Mission = value WHERE attribute = 'membership'",
+        memberships=["Apollo 11"],
+    )
+    assert res.rowcount == 1
+
+
 if __name__ == "__main__":  # pragma: no cover
     from tests.tools import run_tests
 
diff --git a/tests/vectors/test_tokenizer.py b/tests/vectors/test_tokenizer.py
new file mode 100644
--- /dev/null
+++ b/tests/vectors/test_tokenizer.py
@@ -0,0 +1,66 @@
+"""
+Test the permissions model is correctly allowing and blocking queries being executed
+
+"""
+
+import os
+import sys
+
+
+sys.path.insert(1, os.path.join(sys.path[0], "../.."))
+
+from opteryx.compiled.functions import tokenize_and_remove_punctuation
+
+
+def test_tokenizer():
+    string = "That's one small step for a man, one giant leap for man-kind."
+    tokenized = tokenize_and_remove_punctuation(string, {b"for"})
+    assert set(tokenized) == {
+        b"that",
+        b"one",
+        b"small",
+        b"step",
+        b"man",
+        b"giant",
+        b"leap",
+        b"man-kind",
+    }, set(tokenized)
+
+    string = "Apollo 11"
+    tokenized = tokenize_and_remove_punctuation(string, {b"for"})
+    assert set(tokenized) == {b"apollo", b"11"}, set(tokenized)
+
+    string = "NASA's well-documented 1969, Apollo11, journey to the moon — marked by Armstrong's famous step — wasn't universally celebrated initially; however, it's since spurred on-going, far-reaching advancements in space exploration."
+    tokenized = tokenize_and_remove_punctuation(
+        string, {b"for", b"it", b"by", b"to", b"the", b"in"}
+    )
+    assert set(tokenized) == {
+        b"nasa",
+        b"journey",
+        b"mark",
+        b"moon",
+        b"space",
+        b"spur",
+        b"wasnt",
+        b"advancement",
+        b"explore",
+        b"famous",
+        b"on-go",
+        b"step",
+        b"universal",
+        b"since",
+        b"apollo11",
+        b"well-document",
+        b"far-reach",
+        b"celebrat",
+        b"armstrong",
+        b"initial",
+        b"however",
+        b"1969",
+    }, set(tokenized)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    from tests.tools import run_tests
+
+    run_tests()