Repository: SINTEF/dlite (language: python)

Issue:
Python test system for storages
Add Python tests for DLite storage plugins (partial resolution of issue #138).

Tests have been added for BSON, PostgreSQL and YAML. Tests for blobs and CSV files remain to be added.

The PostgreSQL test avoids the need for a PostgreSQL database server. The plugin has been tested against such a server (and passed after a minor modification). The test works by testing against files rather than against a database. The loading tests use two database dump files as input, and the saving tests compare the SQL commands that would have been executed by a server, with those used to generate the instances contained in the two dump files.

CMake files have been added/modified to make these tests run with the existing test system. Thanks to Jesper for help with these files.
Make DLiteStorageBase and DLiteMappingBase available from the python api
Make dlite storage and mapping plugin less magic by subclassing from `dlite.DLiteStorageBase` and `dlite.DLiteStorageBase` instead of magically add these classes to the namespace when loading a Python plugin.



Discussion / hints:

Fix patch:
diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt
index bdc9fe829..8a8457584 100644
--- a/bindings/python/CMakeLists.txt
+++ b/bindings/python/CMakeLists.txt
@@ -51,6 +51,7 @@ include_directories(
   ${Python3_INCLUDE_DIRS}
   ${Python3_NumPy_INCLUDE_DIRS}
   ${dlite-src_SOURCE_DIR}
+  ${dlite-src_SOURCE_DIR}/pyembed
   ${dlite-src_BINARY_DIR}
   ${dlite-src-utils_BINARY_DIR}
   )
diff --git a/bindings/python/dlite-python.i b/bindings/python/dlite-python.i
index 773cab75e..30368b831 100644
--- a/bindings/python/dlite-python.i
+++ b/bindings/python/dlite-python.i
@@ -12,6 +12,7 @@
 #include "floats.h"
 #include "strutils.h"
 #include "dlite.h"
+#include "dlite-python-singletons.h"
 
 #define DLITE_INSTANCE_CAPSULE_NAME ((char *)"dlite.Instance")
 #define DLITE_DATA_CAPSULE_NAME ((char *)"dlite.data")
@@ -1287,6 +1288,12 @@ int dlite_swig_set_property_by_index(DLiteInstance *inst, int i, obj_t *obj)
   }
 }
 
+/* Wrap storage and mapping base */
+%rename(_get_storage_base) dlite_python_storage_base;
+%rename(_get_mapping_base) dlite_python_mapping_base;
+PyObject *dlite_python_storage_base(void);
+PyObject *dlite_python_mapping_base(void);
+
 
 /* ------------------
  * Expose generic api
@@ -1296,12 +1303,6 @@ PyObject *_get_DLiteError(void);
 
 %pythoncode %{
   DLiteError = _dlite._get_DLiteError()
-  
-  class DLiteStorageBase:
-      __name__ = 'DLiteStorageBase'
-      __module__ = 'dlite'
-
-  class DLiteMappingBase:
-      __name__ = 'DLiteMappingBase'
-      __module__ = 'dlite'
+  DLiteStorageBase = _dlite._get_storage_base()
+  DLiteMappingBase = _dlite._get_mapping_base()
 %}
diff --git a/src/pyembed/CMakeLists.txt b/src/pyembed/CMakeLists.txt
index fd0928b5f..30c702a13 100644
--- a/src/pyembed/CMakeLists.txt
+++ b/src/pyembed/CMakeLists.txt
@@ -2,6 +2,7 @@
 
 set(pyembed_sources
   pyembed/dlite-pyembed.c
+  pyembed/dlite-python-singletons.c
   pyembed/dlite-python-storage.c
   pyembed/dlite-python-mapping.c
   PARENT_SCOPE
diff --git a/src/pyembed/dlite-pyembed.c b/src/pyembed/dlite-pyembed.c
index 22f2aaf7b..448063372 100644
--- a/src/pyembed/dlite-pyembed.c
+++ b/src/pyembed/dlite-pyembed.c
@@ -5,6 +5,7 @@
 #include "dlite-pyembed.h"
 #include "dlite-python-storage.h"
 #include "dlite-python-mapping.h"
+#include "dlite-python-singletons.h"
 #include "config-paths.h"
 
 /* Get rid of MSVS warnings */
@@ -343,40 +344,22 @@ DLiteInstance *dlite_pyembed_get_instance(PyObject *pyinst)
   This function loads all Python modules found in `paths` and returns
   a list of plugin objects.
 
-  A Python plugin is a subclass of `baseclassname` that implements the
+  A Python plugin is a subclass of `baseclass` that implements the
   expected functionality.
 
   Returns NULL on error.
  */
-PyObject *dlite_pyembed_load_plugins(FUPaths *paths, const char *baseclassname)
+PyObject *dlite_pyembed_load_plugins(FUPaths *paths, PyObject *baseclass)
 {
-  char initcode[96];
   const char *path;
-  PyObject *baseclass=NULL, *main_module=NULL, *main_dict=NULL;
-  PyObject *ppath=NULL, *pfun=NULL, *subclasses=NULL, *lst=NULL;
+  PyObject *main_dict, *ppath=NULL, *pfun=NULL, *subclasses=NULL, *lst=NULL;
   PyObject *subclassnames=NULL;
   FUIter *iter;
   int i;
 
   dlite_errclr();
   dlite_pyembed_initialise();
-
-  /* Inject base class into the __main__ module */
-  if (snprintf(initcode, sizeof(initcode), "class %s: pass\n",
-               baseclassname) < 0)
-    FAIL("failure to create initialisation code for embedded Python "
-         "__main__ module");
-  if (PyRun_SimpleString(initcode))
-    FAIL("failure when running embedded Python __main__ module "
-         "initialisation code");
-
-  /* Extract the base class from the __main__ module */
-  if (!(main_module = PyImport_AddModule("__main__")))
-    FAIL("cannot load the embedded Python __main__ module");
-  if (!(main_dict = PyModule_GetDict(main_module)))
-    FAIL("cannot access __dict__ of the embedded Python __main__ module");
-  if (!(baseclass = PyDict_GetItemString(main_dict, baseclassname)))
-    FAIL1("cannot get base class '%s' from the main dict", baseclassname);
+  if (!(main_dict = dlite_python_maindict())) goto fail;
 
   /* Get list of initial subclasses and corresponding set subclassnames */
   if ((pfun = PyObject_GetAttrString(baseclass, "__subclasses__")))
diff --git a/src/pyembed/dlite-pyembed.h b/src/pyembed/dlite-pyembed.h
index 83a373d0e..70a0256b1 100644
--- a/src/pyembed/dlite-pyembed.h
+++ b/src/pyembed/dlite-pyembed.h
@@ -106,7 +106,7 @@ DLiteInstance *dlite_pyembed_get_instance(PyObject *pyinst);
 
   Returns NULL on error.
  */
-PyObject *dlite_pyembed_load_plugins(FUPaths *paths, const char *baseclassname);
+PyObject *dlite_pyembed_load_plugins(FUPaths *paths, PyObject *baseclass);
 
 
 #endif /* _DLITE_PYEMBED_H */
diff --git a/src/pyembed/dlite-python-mapping.c b/src/pyembed/dlite-python-mapping.c
index e5a0eca83..460b3dc25 100644
--- a/src/pyembed/dlite-python-mapping.c
+++ b/src/pyembed/dlite-python-mapping.c
@@ -3,6 +3,7 @@
  */
 #include <Python.h>
 #include <assert.h>
+#include <stdlib.h>
 
 /* Python pulls in a lot of defines that conflicts with utils/config.h */
 #define SKIP_UTILS_CONFIG_H
@@ -15,6 +16,7 @@
 #include "dlite-misc.h"
 #include "dlite-mapping-plugins.h"
 #include "dlite-pyembed.h"
+#include "dlite-python-singletons.h"
 #include "dlite-python-mapping.h"
 
 
@@ -133,6 +135,9 @@ void *dlite_python_mapping_load(void)
   const char *path;
   const unsigned char *hash;
   sha3_context c;
+  PyObject *mappingbase;
+
+  if (!(mappingbase = dlite_python_mapping_base())) return NULL;
   if (!(paths = dlite_python_mapping_paths())) return NULL;
   if (!(iter = fu_pathsiter_init(paths, "*.py"))) return NULL;
   sha3_Init256(&c);
@@ -144,7 +149,7 @@ void *dlite_python_mapping_load(void)
              sizeof(mapping_plugin_path_hash)) != 0) {
     if (loaded_mappings) dlite_python_mapping_unload();
     loaded_mappings = dlite_pyembed_load_plugins((FUPaths *)paths,
-                                                 "DLiteMappingBase");
+                                                 mappingbase);
     memcpy(mapping_plugin_path_hash, hash, sizeof(mapping_plugin_path_hash));
   }
   return (void *)loaded_mappings;
diff --git a/src/pyembed/dlite-python-mapping.h b/src/pyembed/dlite-python-mapping.h
index 0ceb7622d..85f84d114 100644
--- a/src/pyembed/dlite-python-mapping.h
+++ b/src/pyembed/dlite-python-mapping.h
@@ -5,6 +5,7 @@
   @file
   @brief Python mappings
 */
+
 #ifdef HAVE_CONFIG_H
 #undef HAVE_CONFIG_H
 #endif
diff --git a/src/pyembed/dlite-python-singletons.c b/src/pyembed/dlite-python-singletons.c
new file mode 100644
index 000000000..05461109c
--- /dev/null
+++ b/src/pyembed/dlite-python-singletons.c
@@ -0,0 +1,69 @@
+#include "dlite-macros.h"
+#include "dlite-pyembed.h"
+#include "dlite-python-singletons.h"
+
+
+/*
+  Returns a pointer to __main__.__dict__ of the embedded interpreater
+  or NULL on error.
+*/
+PyObject *dlite_python_maindict(void)
+{
+  PyObject *main_module, *main_dict=NULL;
+
+  dlite_pyembed_initialise();
+
+  if (!(main_module = PyImport_AddModule("__main__")))
+    FAIL("cannot load the embedded Python __main__ module");
+  if (!(main_dict = PyModule_GetDict(main_module)))
+    FAIL("cannot access __dict__ of the embedded Python __main__ module");
+ fail:
+  return main_dict;
+}
+
+
+/*
+  Creates a new empty singleton class and return it.
+
+  The name of the new class is `classname`.
+
+  Returns NULL on error.
+ */
+PyObject *dlite_python_mainclass(const char *classname)
+{
+  PyObject *main_dict, *class;
+  char initcode[96];
+
+  if (!(main_dict = dlite_python_maindict())) goto fail;
+
+  /* Return newclass if it is already in __main__.__dict__ */
+  if ((class = PyDict_GetItemString(main_dict, classname)))
+    return class;
+
+  /* Create and inject new class into the __main__ module */
+  if (snprintf(initcode, sizeof(initcode), "class %s: pass\n", classname) < 0)
+    FAIL("failure to create initialisation code for embedded Python "
+         "__main__ module");
+  if (PyRun_SimpleString(initcode))
+    FAIL("failure when running embedded Python __main__ module "
+         "initialisation code");
+  if ((class = PyDict_GetItemString(main_dict, classname)))
+    return class;
+
+ fail:
+  return NULL;
+}
+
+
+/* Returns the base class for storage plugins. */
+PyObject *dlite_python_storage_base(void)
+{
+  return dlite_python_mainclass("DLiteStorageBase");
+}
+
+
+/* Returns the base class for mapping plugins. */
+PyObject *dlite_python_mapping_base(void)
+{
+  return dlite_python_mainclass("DLiteMappingBase");
+}
diff --git a/src/pyembed/dlite-python-singletons.h b/src/pyembed/dlite-python-singletons.h
new file mode 100644
index 000000000..e07435e72
--- /dev/null
+++ b/src/pyembed/dlite-python-singletons.h
@@ -0,0 +1,38 @@
+#ifndef _DLITE_PYTHON_SINGLETONS_H
+#define _DLITE_PYTHON_SINGLETONS_H
+
+/**
+  @file
+  @brief Python singletons
+*/
+#include <Python.h>
+
+
+/**
+  Returns a pointer to __main__.__dict__ of the embedded interpreater
+  or NULL on error.
+*/
+PyObject *dlite_python_maindict(void);
+
+/**
+  Creates a new empty singleton class and return it.
+
+  The name of the new class is `classname`.
+
+  Returns NULL on error.
+ */
+PyObject *dlite_python_mainclass(const char *classname);
+
+
+/**
+  Returns the base class for storage plugins.
+*/
+PyObject *dlite_python_storage_base(void);
+
+/**
+  Returns the base class for mapping plugins.
+*/
+PyObject *dlite_python_mapping_base(void);
+
+
+#endif /* _DLITE_PYTHON_SINGLETONS_H */
diff --git a/src/pyembed/dlite-python-storage.c b/src/pyembed/dlite-python-storage.c
index 31360af0d..c718e991a 100644
--- a/src/pyembed/dlite-python-storage.c
+++ b/src/pyembed/dlite-python-storage.c
@@ -15,6 +15,7 @@
 #include "dlite-misc.h"
 #include "dlite-storage-plugins.h"
 #include "dlite-pyembed.h"
+#include "dlite-python-singletons.h"
 #include "dlite-python-storage.h"
 
 #define GLOBALS_ID "dlite-python-storage-globals"
@@ -172,13 +173,18 @@ const char **dlite_python_storage_paths_get(void)
 */
 void *dlite_python_storage_load(void)
 {
+  PyObject *storagebase;
   PythonStorageGlobals *g = get_globals();
+
+  if (!(storagebase = dlite_python_storage_base())) return NULL;
+
   if (!g->loaded_storages || g->modified) {
     const FUPaths *paths;
     if (g->loaded_storages) dlite_python_storage_unload();
     if (!(paths = dlite_python_storage_paths())) return NULL;
+
     g->loaded_storages = dlite_pyembed_load_plugins((FUPaths *)paths,
-                                                 "DLiteStorageBase");
+                                                    storagebase);
   }
   return (void *)g->loaded_storages;
 }
diff --git a/src/pyembed/dlite-python-storage.h b/src/pyembed/dlite-python-storage.h
index ed47e17f5..6082f497f 100644
--- a/src/pyembed/dlite-python-storage.h
+++ b/src/pyembed/dlite-python-storage.h
@@ -62,5 +62,10 @@ void *dlite_python_storage_load(void);
 void dlite_python_storage_unload(void);
 
 
+/**
+  Returns the base class for storage plugins.
+*/
+PyObject *dlite_python_storage_base(void);
+
 
 #endif /* _DLITE_PYTHON_STORAGE_H */


Test patch:
diff --git a/bindings/python/tests/test_python_storage.py b/bindings/python/tests/test_python_storage.py
index e2361c3ec..44a862e70 100755
--- a/bindings/python/tests/test_python_storage.py
+++ b/bindings/python/tests/test_python_storage.py
@@ -2,6 +2,20 @@
 
 import dlite
 
+try:
+    import bson
+except ImportError:
+    HAVE_BSON = False
+else:
+    HAVE_BSON = True
+
+try:
+    import yaml
+except ImportError:
+    HAVE_YAML = False
+else:
+    HAVE_YAML = True
+
 
 thisdir = os.path.dirname(__file__)
 
@@ -42,24 +56,10 @@
 # =====================================================================
 # Test the BSON and YAML Python plugins
 
-def ignore_error(name):
-    print(f"Warning: Ignored error while test importing '{name}'")
-
-import pkgutil
-have_bson = False # Do we have pymongo?
-have_yaml = False # Do we have PyYAML?
-for pkg in pkgutil.walk_packages(onerror=ignore_error):
-    if not have_bson and pkg.name == 'pymongo':
-        have_bson = True
-    elif not have_yaml and pkg.name == 'yaml':
-        have_yaml = True
-    if have_bson and have_yaml:
-        break
-
 input_dir = thisdir.replace('bindings', 'storages')
 input_dir = input_dir.replace('tests', 'tests-python/input/')
 
-if have_bson:
+if HAVE_BSON:
     # Test BSON
     print('\n\n=== Test BSON plugin ===')
     meta_file = input_dir + 'test_meta.bson'
@@ -104,9 +104,9 @@ def ignore_error(name):
     os.remove(data_test_file)
     del inst1, inst2, inst3, inst4
 else:
-    print('Skip testing BSON plugin - pymongo not installed')
+    print('Skip testing BSON plugin - bson not installed')
 
-if have_yaml:
+if HAVE_YAML:
     # Test YAML
     print('\n\n=== Test YAML plugin ===')
     meta_file = input_dir + 'test_meta.yaml'
diff --git a/src/tests/python/test_pyembed.c b/src/tests/python/test_pyembed.c
index 5043c503a..77de690e4 100644
--- a/src/tests/python/test_pyembed.c
+++ b/src/tests/python/test_pyembed.c
@@ -6,6 +6,7 @@
 #include "dlite.h"
 #include "dlite-macros.h"
 #include "dlite-pyembed.h"
+#include "dlite-python-singletons.h"
 
 
 typedef DLiteInstance *(*fun_t)(const char *id);
@@ -22,11 +23,12 @@ MU_TEST(test_load_modules)
   int i;
   FUPaths paths;
   PyObject *plugins;
+  PyObject *mappingbase = dlite_python_mapping_base();
 
   fu_paths_init(&paths, "DLITE_PYTHON_MAPPING_PLUGIN_DIRS");
   fu_paths_insert(&paths, STRINGIFY(TESTDIR), 0);
 
-  plugins = dlite_pyembed_load_plugins(&paths, "DLiteMappingBase");
+  plugins = dlite_pyembed_load_plugins(&paths, mappingbase);
   mu_check(plugins);
   mu_check(PyList_Check(plugins));