Repository: MongoEngine/mongoengine

Issue:
MongoDB 4.4 / pymongo 3.11 Building a tag cloud with item_frequencies fails
I have the default code from the docs to build a tag cloud. It started failing with MongoDB 4.4
Tried mongoengine 18.2 and latest 22.1

```  
  tag_freqs = queryset.item_frequencies('tags', normalize=True)
  File "/opt/app-root/lib/python3.6/site-packages/mongoengine/queryset/base.py", line 1474, in item_frequencies
    normalize=normalize)
  File "/opt/app-root/lib/python3.6/site-packages/mongoengine/queryset/base.py", line 1661, in _item_frequencies_map_reduce
    for f in values:
  File "/opt/app-root/lib/python3.6/site-packages/mongoengine/queryset/base.py", line 1326, in map_reduce
    map_f, reduce_f, **mr_args)
  File "/opt/app-root/lib/python3.6/site-packages/pymongo/collection.py", line 2896, in inline_map_reduce
    self.read_preference, **kwargs)
  File "/opt/app-root/lib/python3.6/site-packages/pymongo/collection.py", line 2789, in _map_reduce
    user_fields=user_fields)
  File "/opt/app-root/lib/python3.6/site-packages/pymongo/collection.py", line 253, in _command
    user_fields=user_fields)
  File "/opt/app-root/lib/python3.6/site-packages/pymongo/pool.py", line 694, in command
    exhaust_allowed=exhaust_allowed)
  File "/opt/app-root/lib/python3.6/site-packages/pymongo/network.py", line 161, in command
    parse_write_concern_error=parse_write_concern_error)
  File "/opt/app-root/lib/python3.6/site-packages/pymongo/helpers.py", line 164, in _check_command_response
    raise OperationFailure(errmsg, code, response, max_wire_version)
pymongo.errors.OperationFailure: 'map' must be of string or code type,
```
Thank you


Discussion:

Resolving patch:
diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py
--- a/mongoengine/mongodb_support.py
+++ b/mongoengine/mongodb_support.py
@@ -8,6 +8,8 @@
 # get_mongodb_version()
 MONGODB_34 = (3, 4)
 MONGODB_36 = (3, 6)
+MONGODB_42 = (4, 2)
+MONGODB_44 = (4, 4)
 
 
 def get_mongodb_version():
diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py
--- a/mongoengine/queryset/base.py
+++ b/mongoengine/queryset/base.py
@@ -1355,21 +1355,18 @@ def map_reduce(
 
         MapReduceDocument = _import_class("MapReduceDocument")
 
-        if not hasattr(self._collection, "map_reduce"):
-            raise NotImplementedError("Requires MongoDB >= 1.7.1")
-
         map_f_scope = {}
         if isinstance(map_f, Code):
             map_f_scope = map_f.scope
             map_f = str(map_f)
-        map_f = Code(queryset._sub_js_fields(map_f), map_f_scope)
+        map_f = Code(queryset._sub_js_fields(map_f), map_f_scope or None)
 
         reduce_f_scope = {}
         if isinstance(reduce_f, Code):
             reduce_f_scope = reduce_f.scope
             reduce_f = str(reduce_f)
         reduce_f_code = queryset._sub_js_fields(reduce_f)
-        reduce_f = Code(reduce_f_code, reduce_f_scope)
+        reduce_f = Code(reduce_f_code, reduce_f_scope or None)
 
         mr_args = {"query": queryset._query}
 
@@ -1379,7 +1376,7 @@ def map_reduce(
                 finalize_f_scope = finalize_f.scope
                 finalize_f = str(finalize_f)
             finalize_f_code = queryset._sub_js_fields(finalize_f)
-            finalize_f = Code(finalize_f_code, finalize_f_scope)
+            finalize_f = Code(finalize_f_code, finalize_f_scope or None)
             mr_args["finalize"] = finalize_f
 
         if scope:


Test patch:
diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py
--- a/tests/document/test_indexes.py
+++ b/tests/document/test_indexes.py
@@ -7,6 +7,7 @@
 
 from mongoengine import *
 from mongoengine.connection import get_db
+from mongoengine.mongodb_support import MONGODB_42, get_mongodb_version
 
 
 class TestIndexes(unittest.TestCase):
@@ -452,9 +453,11 @@ class Test(Document):
             .get("stage")
             == "IXSCAN"
         )
+        mongo_db = get_mongodb_version()
+        PROJECTION_STR = "PROJECTION" if mongo_db < MONGODB_42 else "PROJECTION_COVERED"
         assert (
             query_plan.get("queryPlanner").get("winningPlan").get("stage")
-            == "PROJECTION"
+            == PROJECTION_STR
         )
 
         query_plan = Test.objects(a=1).explain()
diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py
--- a/tests/queryset/test_queryset.py
+++ b/tests/queryset/test_queryset.py
@@ -21,7 +21,10 @@
     QuerySetManager,
     queryset_manager,
 )
-from tests.utils import requires_mongodb_gte_44
+from tests.utils import (
+    requires_mongodb_gte_44,
+    requires_mongodb_lt_42,
+)
 
 
 class db_ops_tracker(query_counter):
@@ -1490,6 +1493,7 @@ class BlogPost(Document):
 
         BlogPost.drop_collection()
 
+    @requires_mongodb_lt_42
     def test_exec_js_query(self):
         """Ensure that queries are properly formed for use in exec_js."""
 
@@ -1527,6 +1531,7 @@ class BlogPost(Document):
 
         BlogPost.drop_collection()
 
+    @requires_mongodb_lt_42
     def test_exec_js_field_sub(self):
         """Ensure that field substitutions occur properly in exec_js functions."""
 
@@ -2660,6 +2665,8 @@ class BlogPost(Document):
             title = StringField(primary_key=True)
             tags = ListField(StringField())
 
+        BlogPost.drop_collection()
+
         post1 = BlogPost(title="Post #1", tags=["mongodb", "mongoengine"])
         post2 = BlogPost(title="Post #2", tags=["django", "mongodb"])
         post3 = BlogPost(title="Post #3", tags=["hitchcock films"])
@@ -2688,12 +2695,15 @@ class BlogPost(Document):
             }
         """
 
-        results = BlogPost.objects.map_reduce(map_f, reduce_f, "myresults")
+        results = BlogPost.objects.order_by("_id").map_reduce(
+            map_f, reduce_f, "myresults2"
+        )
         results = list(results)
 
-        assert results[0].object == post1
-        assert results[1].object == post2
-        assert results[2].object == post3
+        assert len(results) == 3
+        assert results[0].object.id == post1.id
+        assert results[1].object.id == post2.id
+        assert results[2].object.id == post3.id
 
         BlogPost.drop_collection()
 
@@ -2701,7 +2711,6 @@ def test_map_reduce_custom_output(self):
         """
         Test map/reduce custom output
         """
-        register_connection("test2", "mongoenginetest2")
 
         class Family(Document):
             id = IntField(primary_key=True)
@@ -2774,6 +2783,7 @@ class Person(Document):
                             family.persons.push(person);
                             family.totalAge += person.age;
                         });
+                        family.persons.sort((a, b) => (a.age > b.age))
                     }
                 });
 
@@ -2802,10 +2812,10 @@ class Person(Document):
             "_id": 1,
             "value": {
                 "persons": [
+                    {"age": 17, "name": "Tayza Mariana"},
                     {"age": 21, "name": "Wilson Jr"},
-                    {"age": 45, "name": "Wilson Father"},
                     {"age": 40, "name": "Eliana Costa"},
-                    {"age": 17, "name": "Tayza Mariana"},
+                    {"age": 45, "name": "Wilson Father"},
                 ],
                 "totalAge": 123,
             },
@@ -2815,9 +2825,9 @@ class Person(Document):
             "_id": 2,
             "value": {
                 "persons": [
+                    {"age": 10, "name": "Igor Gabriel"},
                     {"age": 16, "name": "Isabella Luanna"},
                     {"age": 36, "name": "Sandra Mara"},
-                    {"age": 10, "name": "Igor Gabriel"},
                 ],
                 "totalAge": 62,
             },
@@ -2827,8 +2837,8 @@ class Person(Document):
             "_id": 3,
             "value": {
                 "persons": [
-                    {"age": 30, "name": "Arthur WA"},
                     {"age": 25, "name": "Paula Leonel"},
+                    {"age": 30, "name": "Arthur WA"},
                 ],
                 "totalAge": 55,
             },
@@ -3109,6 +3119,7 @@ class Person(Document):
         freq = Person.objects.item_frequencies("city", normalize=True, map_reduce=True)
         assert freq == {"CRB": 0.5, None: 0.5}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_with_null_embedded(self):
         class Data(EmbeddedDocument):
             name = StringField()
@@ -3137,6 +3148,7 @@ class Person(Document):
         ot = Person.objects.item_frequencies("extra.tag", map_reduce=True)
         assert ot == {None: 1.0, "friend": 1.0}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_with_0_values(self):
         class Test(Document):
             val = IntField()
@@ -3151,6 +3163,7 @@ class Test(Document):
         ot = Test.objects.item_frequencies("val", map_reduce=False)
         assert ot == {0: 1}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_with_False_values(self):
         class Test(Document):
             val = BooleanField()
@@ -3165,6 +3178,7 @@ class Test(Document):
         ot = Test.objects.item_frequencies("val", map_reduce=False)
         assert ot == {False: 1}
 
+    @requires_mongodb_lt_42
     def test_item_frequencies_normalize(self):
         class Test(Document):
             val = IntField()
@@ -3551,7 +3565,8 @@ class Book(Document):
         Book.objects.create(title="The Stories", authors=[mark_twain, john_tolkien])
 
         authors = Book.objects.distinct("authors")
-        assert authors == [mark_twain, john_tolkien]
+        authors_names = {author.name for author in authors}
+        assert authors_names == {mark_twain.name, john_tolkien.name}
 
     def test_distinct_ListField_EmbeddedDocumentField_EmbeddedDocumentField(self):
         class Continent(EmbeddedDocument):
@@ -3588,7 +3603,8 @@ class Book(Document):
         assert country_list == [scotland, tibet]
 
         continent_list = Book.objects.distinct("authors.country.continent")
-        assert continent_list == [europe, asia]
+        continent_list_names = {c.continent_name for c in continent_list}
+        assert continent_list_names == {europe.continent_name, asia.continent_name}
 
     def test_distinct_ListField_ReferenceField(self):
         class Bar(Document):
diff --git a/tests/utils.py b/tests/utils.py
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -34,6 +34,10 @@ def get_as_pymongo(doc):
     return doc.__class__.objects.as_pymongo().get(id=doc.id)
 
 
+def requires_mongodb_lt_42(func):
+    return _decorated_with_ver_requirement(func, (4, 2), oper=operator.lt)
+
+
 def requires_mongodb_gte_44(func):
     return _decorated_with_ver_requirement(func, (4, 4), oper=operator.ge)