Repository: SINTEF/dlite (language: python)

Issue:
Fix iterate over instances/uuids in storage
`storage.instances()` and `storage.get_uuids()` works. But calling these functions with a metadata uri for filtering the result results in an error - at least for the mongodb backend...



Discussion / hints:

Fix patch:
diff --git a/bindings/python/dlite-storage-python.i b/bindings/python/dlite-storage-python.i
index ff23d65c6..1f6ce4c91 100644
--- a/bindings/python/dlite-storage-python.i
+++ b/bindings/python/dlite-storage-python.i
@@ -45,11 +45,10 @@
           """Create a new storage from `url`."""
           return cls(url)
 
-
       def instances(self, pattern=None):
           """Returns an iterator over all instances in storage whos
           metadata URI matches `pattern`."""
-          return StorageIterator(pattern)
+          return StorageIterator(self, pattern=pattern)
 
       def load(self, id, metaid=None):
           """Loads instance `id` from this storage and return it.
diff --git a/bindings/python/dlite-storage.i b/bindings/python/dlite-storage.i
index c43ffc78c..b421371b6 100644
--- a/bindings/python/dlite-storage.i
+++ b/bindings/python/dlite-storage.i
@@ -34,8 +34,9 @@ instances whos metadata URI matches `pattern` are returned.
     dlite_storage_iter_free($self->s, $self->state);
     free($self);
   }
+
   %feature("docstring", "\
-Returns UUID of next instance or None if exhausted.") next;
+Returns next instance or None if exhausted.") next;
   %newobject next;
   struct _DLiteInstance *next(void) {
     char uuid[DLITE_UUID_LENGTH+1];
@@ -43,6 +44,7 @@ Returns UUID of next instance or None if exhausted.") next;
       return dlite_instance_load($self->s, uuid);
     return NULL;
   }
+
   struct StorageIterator *__iter__(void) {
     return $self;
   }
diff --git a/requirements.txt b/requirements.txt
index 4238e9103..e35a3bdb7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,3 +7,4 @@ pymongo
 rdflib
 pint
 openpyxl
+pymongo
diff --git a/requirements_dev.txt b/requirements_dev.txt
index 8c8321d3f..c453c0b5f 100644
--- a/requirements_dev.txt
+++ b/requirements_dev.txt
@@ -1,3 +1,4 @@
 ipython==7.31.1
 ipykernel==6.0.1
 ipython_genutils==0.2.0
+mongomock==4.1.2
diff --git a/storages/python/dlite-plugins-python.c b/storages/python/dlite-plugins-python.c
index 09b50fbca..217f9633d 100644
--- a/storages/python/dlite-plugins-python.c
+++ b/storages/python/dlite-plugins-python.c
@@ -221,7 +221,6 @@ void *iterCreate(const DLiteStorage *s, const char *pattern)
   void *retval=NULL;
   Iter *iter = NULL;
   PyObject *class = (PyObject *)s->api->data;
-  PyObject *patt = NULL;
   const char *classname;
   dlite_errclr();
   if (!(classname = dlite_pyembed_classname(class)))
@@ -235,11 +234,6 @@ void *iterCreate(const DLiteStorage *s, const char *pattern)
   if (!PyIter_Check(iter->v))
     FAIL1("method %s.queue() does not return a iterator object", classname);
 
-  if (pattern) {
-    patt = PyUnicode_FromString(pattern);
-    PyObject_SetAttrString(iter->v, "pattern", patt);
-  }
-  //iter->pattern = pattern;
   iter->classname = classname;
 
   retval = (void *)iter;
@@ -261,7 +255,8 @@ int iterNext(void *iter, char *buf)
   int retval = -1;
   Iter *i = (Iter *)iter;
   PyObject *next = PyIter_Next((PyObject *)i->v);
-  if (dlite_pyembed_err_check("error iteratine over %s.queue()",
+
+  if (dlite_pyembed_err_check("error iterating over %s.queue()",
                               i->classname)) goto fail;
   if (next) {
     if (!PyUnicode_Check(next))
diff --git a/storages/python/python-storage-plugins/mongodb.py b/storages/python/python-storage-plugins/mongodb.py
new file mode 100644
index 000000000..184bf3c9a
--- /dev/null
+++ b/storages/python/python-storage-plugins/mongodb.py
@@ -0,0 +1,92 @@
+import json
+import sys
+from urllib.parse import quote_plus, urlparse
+
+import pymongo
+
+import dlite
+from dlite.options import Options
+from dlite.utils import instance_from_dict
+
+
+class mongodb(dlite.DLiteStorageBase):
+    """DLite storage plugin for PostgreSQL."""
+    def open(self, uri, options=None):
+        """Opens `uri`.
+
+        The `options` argument provies additional input to the driver.
+        Which options that are supported varies between the plugins.  It
+        should be a valid URL query string of the form:
+
+            key1=value1;key2=value2...
+
+        An ampersand (&) may be used instead of the semicolon (;).
+
+        Options:
+        - database: Name of database to use (default: "dlite")
+        - collection: Name of collection to use (default: "dlite_coll")
+        - user: User name.
+        - password: Password.
+        - mode: "r" for opening in read-only mode, "w" for read/write mode.
+        - authMechanism: Authentication mechanism (default: "SCRAM-SHA-256")
+        - mock: Whether to use mongomock.
+        """
+        opts = Options(
+            options,
+            defaults='database=dlite;collection=dlite_coll;mode=w;'
+            'authMechanism=SCRAM-SHA-256;mock=no',
+        )
+        opts.setdefault('password', None)
+        self.writable = True if 'w' in opts.mode else False
+
+        # Connect to existing database
+        user = quote_plus(opts.user) if opts.user else None
+        password = quote_plus(opts.password) if opts.password else None
+
+        if dlite.asbool(opts.mock):
+            import mongomock
+            mongo_url = urlparse(f'mongodb://{uri}')
+            port = mongo_url.port if mongo_url.port else 27017
+
+            @mongomock.patch(servers=((mongo_url.hostname, port), ))
+            def get_client():
+                return open_client()
+        else:
+            def get_client():
+                return open_client()
+
+        def open_client():
+            client = pymongo.MongoClient(
+                host=uri,
+                username=user,
+                password=password,
+                authSource=opts.database,
+                authMechanism=opts.authMechanism,
+            )
+            return client
+
+        self.client = get_client()
+        self.collection = self.client[opts.database][opts.collection]
+        self.options = opts
+
+    def close(self):
+        """Closes this storage."""
+        self.client.close()
+
+    def load(self, id):
+        """Loads `id` from current storage and return it as a new instance."""
+        uuid = dlite.get_uuid(id)
+        document = self.collection.find_one({'uuid': uuid})
+        return instance_from_dict(document)
+
+    def save(self, inst):
+        """Stores `inst` in current storage."""
+        document = inst.asdict(uuid=True)
+        self.collection.insert_one(document)
+
+    def queue(self, pattern=None):
+        """Generator method that iterates over all UUIDs in the storage
+        who's metadata URI matches glob pattern `pattern`."""
+        filter = {"meta": pattern} if pattern else {}
+        for doc in self.collection.find(filter=filter, projection=["uuid"]):
+            yield doc["uuid"]


Test patch:
diff --git a/storages/python/tests-python/CMakeLists.txt b/storages/python/tests-python/CMakeLists.txt
index 560198343..cc808d976 100644
--- a/storages/python/tests-python/CMakeLists.txt
+++ b/storages/python/tests-python/CMakeLists.txt
@@ -5,6 +5,7 @@ set(python-tests
   test_bson_storage_python
   test_postgresql_storage_python
   test_yaml_storage_python
+  test_mongodb_python
   )
 
 include(FindPythonModule)
@@ -62,4 +63,7 @@ foreach(test ${python-tests})
     ENVIRONMENT "LD_LIBRARY_PATH=${dlite_LD_LIBRARY_PATH_NATIVE}")
   endif()
 
+  # Skip tests that exit with return code 44
+  set_property(TEST ${test} PROPERTY SKIP_RETURN_CODE 44)
+
 endforeach()
diff --git a/storages/python/tests-python/input/test_data.json b/storages/python/tests-python/input/test_data.json
new file mode 100644
index 000000000..e791e38ad
--- /dev/null
+++ b/storages/python/tests-python/input/test_data.json
@@ -0,0 +1,35 @@
+{
+  "2f8ba28c-add6-5718-a03c-ea46961d6ca7":   {
+    "uri": "my_test_instance_1",
+    "meta": "http://onto-ns.com/meta/0.1/TestEntity",
+    "dimensions": {
+      "L": 2,
+      "M": 1,
+      "N": 3
+    },
+    "properties": {
+      "myblob": "0a0010",
+      "mydouble": 3.14,
+      "myfixstring": "Al",
+      "mystring": "allocated string...",
+      "myshort": 17,
+      "myarray": [[[1, 2, 3]], [[4, 5, 6]]]
+    }
+  },
+  "52522ba5-6bfe-4a64-992d-e9ec4080fbac": {
+    "meta": "http://onto-ns.com/meta/0.1/TestEntity",
+    "dimensions": {
+      "L": 2,
+      "M": 2,
+      "N": 3
+    },
+    "properties": {
+      "myblob": "0a0120",
+      "mydouble": 2.72,
+      "myfixstring": "Si",
+      "mystring": "...",
+      "myshort": 13,
+      "myarray": [[[1, 2, 3], [3, 2, 1]], [[4, 5, 6], [6, 5, 4]]]
+    }
+  }
+}
diff --git a/storages/python/tests-python/input/test_meta.json b/storages/python/tests-python/input/test_meta.json
new file mode 100644
index 000000000..59b23f7c0
--- /dev/null
+++ b/storages/python/tests-python/input/test_meta.json
@@ -0,0 +1,51 @@
+{
+  "uri": "http://onto-ns.com/meta/0.1/TestEntity",
+  "description": "test entity with explicit meta",
+  "dimensions": [
+    {
+      "name": "L",
+      "description": "first dim"
+    },
+    {
+      "name": "M",
+      "description": "second dim"
+    },
+    {
+      "name": "N",
+      "description": "third dim"
+    }
+  ],
+  "properties": [
+    {
+      "name": "myblob",
+      "type": "blob3"
+    },
+    {
+      "name": "mydouble",
+      "type": "float64",
+      "unit": "m",
+      "description": "A double following a single character..."
+    },
+    {
+      "name": "myfixstring",
+      "type": "string3",
+      "description": "A fix string."
+    },
+    {
+      "name": "mystring",
+      "type": "string",
+      "description": "A string pointer."
+    },
+    {
+      "name": "myshort",
+      "type": "uint16",
+      "description": "An unsigned short integer."
+    },
+    {
+      "name": "myarray",
+      "type": "int32",
+      "dims": ["L", "M", "N"],
+      "description": "An array string pointer."
+    }
+  ]
+}
diff --git a/storages/python/tests-python/test_mongodb_python.py b/storages/python/tests-python/test_mongodb_python.py
new file mode 100644
index 000000000..2731e3593
--- /dev/null
+++ b/storages/python/tests-python/test_mongodb_python.py
@@ -0,0 +1,64 @@
+"""Script to test the 'mongodb' DLite plugin from Python."""
+import sys
+from pathlib import Path
+
+import dlite
+
+try:
+    import mongomock
+except ImportError:
+    sys.exit(44)  # skip test
+
+
+thisdir = Path(__file__).absolute().parent
+inputdir = thisdir / "input"
+
+# Load existing test data
+meta = dlite.Instance.from_location("json", inputdir / "test_meta.json")
+inst1 = dlite.Instance.from_location("json", inputdir / "test_data.json",
+                                     id="52522ba5-6bfe-4a64-992d-e9ec4080fbac")
+inst2 = dlite.Instance.from_location("json", inputdir / "test_data.json",
+                                     id="2f8ba28c-add6-5718-a03c-ea46961d6ca7")
+
+# Create storage
+storage = dlite.Storage(
+    "mongodb", "localhost", "user=testuser;password=testpw;mock=true"
+)
+
+# Store metadata and data
+meta.save(storage)
+inst1.save(storage)
+inst2.save(storage)
+
+# Remove existing instances from dlite memory cache to ensure that we
+# we are reading them from storage and not just from the cache...
+# (This isn't sufficient for the metadata though...)
+uuid0 = meta.uuid
+uuid1 = inst1.uuid
+uuid2 = inst2.uuid
+del inst1
+del inst2
+del meta
+
+# Load metadata and data
+new_meta = storage.load(id=uuid0)
+new_inst1 = storage.load(id=uuid1)
+new_inst2 = storage.load(id=uuid2)
+
+# Simple queries
+instances = list(storage.instances())
+assert len(instances) == 3
+assert new_meta in instances
+assert new_inst1 in instances
+assert new_inst2 in instances
+meta, = storage.instances(dlite.ENTITY_SCHEMA)
+assert meta.uuid == uuid0
+insts = storage.instances(meta.uri)
+assert set(insts) == set([new_inst1, new_inst2])
+
+uuids = storage.get_uuids()
+assert len(uuids) == 3
+meta_uuid, = storage.get_uuids(dlite.ENTITY_SCHEMA)
+assert meta_uuid == uuid0
+inst_uuids = storage.get_uuids(meta.uri)
+assert set(inst_uuids) == set([uuid1, uuid2])