Repository: SINTEF/dlite (language: python)

Issue:
Instantiate properties from dict
Make it possible to instantiate properties from dict. An important use case is to make it easier to work with dicts, dataclasses and pydantic datamodels. We should make it easy to convert to/from nested dicts and instances with ref-type properties.

This could be exposed via:
- A `dlite.Instance.set_dict(dict_obj)` method that sets properties of an instance from a dict.
- A `dlite.Instance.get_dict()` method that returns the properties as a dict.
- Instantiation of Metadata with ref-type properties from nested dicts, like
  ```python
  Entity = dlite.get_instance("http://onto-ns.com/meta/0.1/Entity")
  Entity(dimensions={...}, properties={"ref_prop": {...}, "other_prop": True, ...})
  ```

Use this to improve the support for dataclasses and pydantic datamodels in utils.py (maybe a separate issue).

---

At C-level we could support this by adding a functions:
```C
int dlite_swig_set_dict(DLiteInstance *inst, obj_t *dict_obj);
```
This function could be exposed directly as the `set_dict()` method, but also used by `dlite_swig_set_scalar()` to support metadata instantiation with ref-type properties from nested dicts.

The `get_dict()` method could probably be implemented directly in Python.


Discussion / hints:

Fix patch:
diff --git a/bindings/python/dlite-entity-python.i b/bindings/python/dlite-entity-python.i
index 815051976..924840590 100644
--- a/bindings/python/dlite-entity-python.i
+++ b/bindings/python/dlite-entity-python.i
@@ -553,7 +553,7 @@ def get_instance(id: "str", metaid: "str" = None, check_storages: "bool" = True)
         return self.uuid == other.uuid
 
     def __str__(self):
-        return self.asjson(indent=2)
+        return self.asjson()
 
     def __reduce__(self):
         # ensures that instances can be pickled
diff --git a/bindings/python/dlite-python.i b/bindings/python/dlite-python.i
index 1e705a904..1e8f16a0b 100644
--- a/bindings/python/dlite-python.i
+++ b/bindings/python/dlite-python.i
@@ -229,14 +229,10 @@ int dlite_swig_read_python_blob(PyObject *src, uint8_t *dest, size_t n)
 {
   int retval=-1;
   if (PyUnicode_Check(src)) {
-    long len;
-    if (PyUnicode_READY(src))
-      FAIL("failed preparing string");
-    len = (long)PyUnicode_GET_LENGTH(src);
-    if (len != (long)n*2)
-      FAIL3("cannot convert Python string of length %d (expected length %d) "
-            "to blob%d", (int)len, (int)n*2, (int)n);
-    if (strhex_decode(dest, n, (char *)PyUnicode_1BYTE_DATA(src), len) < 0)
+    Py_ssize_t len;
+    const char *s = PyUnicode_AsUTF8AndSize(src, &len);
+    if (!s) FAIL("failed representing string as UTF-8");
+    if (strhex_decode(dest, n, s, len) < 0)
       FAIL("cannot convert Python string to blob");
     retval = len/2;
   } else if (PyObject_CheckBuffer(src)) {
@@ -457,12 +453,11 @@ int dlite_swig_set_array(void *ptr, int ndims, int *dims,
           if (p[i]) free(p[i]);
         } else if (PyUnicode_Check(s)) {
           assert(s);
-          if (PyUnicode_READY(s))
-            FAIL("failed preparing string");
-          long len = (long)PyUnicode_GET_LENGTH(s);
+          Py_ssize_t len;
+          const char *str = PyUnicode_AsUTF8AndSize(s, &len);
+          if (!str) FAIL("failed representing string as UTF-8");
           p[i] = realloc(p[i], len+1);
-          memcpy(p[i], PyUnicode_1BYTE_DATA(s), len);
-          p[i][len] = '\0';
+          memcpy(p[i], str, len+1);
         } else {
           FAIL("expected None or unicode elements");
           Py_DECREF(s);
@@ -777,53 +772,50 @@ int dlite_swig_set_scalar(void *ptr, DLiteType type, size_t size, obj_t *obj)
 
   case dliteFixString:
     {
-      size_t n;
-      Py_UCS1 *s;
+      Py_ssize_t n;
       PyObject *str = PyObject_Str(obj);
-      if (!str) FAIL("cannot convert to string");
-      if (PyUnicode_READY(str)) {
-        Py_DECREF(str);
-        FAIL("failed preparing string");
-      }
-      n = PyUnicode_GET_LENGTH(str);
-      if (n > size) {
-        Py_DECREF(str);
+      const char *s = PyUnicode_AsUTF8AndSize(str, &n);
+      Py_DECREF(str);
+      if (!s) FAIL("cannot represent string as UTF-8");
+      if (n >= (Py_ssize_t)size)
         FAIL2("Length of string is %lu. Exceeds available size: %lu",
               (unsigned long)n, (unsigned long)size);
-      }
-      s = PyUnicode_1BYTE_DATA(str);
-      memset(ptr, 0, size);
-      memcpy(ptr, s, n);
-      Py_DECREF(str);
+      memcpy(ptr, s, n+1);
     }
     break;
 
   case dliteStringPtr:
     {
-      size_t n;
-      unsigned char *s;
       char *p;
+      Py_ssize_t n;
       PyObject *str = PyObject_Str(obj);
-      if (!str) FAIL("cannot convert to string");
-      if (PyUnicode_READY(str)) {
-        Py_DECREF(str);
-        FAIL("failed preparing string");
-      }
-      n = PyUnicode_GET_LENGTH(str);
-      s = PyUnicode_1BYTE_DATA(str);
-      *(void **)ptr = realloc(*(void **)ptr, n+1);
-      p = *((char **)ptr);
-      memcpy(p, s, n);
-      p[n] = '\0';
+      const char *s = PyUnicode_AsUTF8AndSize(str, &n);
       Py_DECREF(str);
+      if (!s) FAIL("cannot represent string as UTF-8");
+      if (!(p = realloc(*(void **)ptr, n+1))) FAIL("allocation failure");
+      *(char **)ptr = p;
+      if (p)
+        memcpy(p, s, n+1);
     }
     break;
 
   case dliteRef:
     {
-      DLiteInstance **q = ptr;
-      if (*q) dlite_instance_decref(*q);
-      *q = dlite_pyembed_get_instance(obj);
+      DLiteInstance *inst=NULL;
+      if (obj == Py_None) {
+        // do nothing here, handled below...
+      } else if (PyUnicode_Check(obj)) {
+        const char *s = PyUnicode_AsUTF8(obj);
+        if (!s) FAIL("cannot represent string as UTF-8");
+        inst = dlite_instance_get(s);
+      } else {
+        inst = dlite_pyembed_get_instance(obj);
+      }
+      if (inst || obj == Py_None) {
+        DLiteInstance **q = ptr;
+        if (*q) dlite_instance_decref(*q);
+        *q = inst;
+      }
     }
     break;
 
@@ -971,8 +963,9 @@ int dlite_swig_set_scalar(void *ptr, DLiteType type, size_t size, obj_t *obj)
         for (i=0; i<n; i++) {
           PyObject *item = PySequence_Fast_GET_ITEM(lst, i);
           assert(PyUnicode_Check(item));
-          PyUnicode_READY(item);
-          s[i] = strdup(PyUnicode_DATA(item));
+          const char *q = PyUnicode_AsUTF8(item);
+          if (!q) FAIL("failed representing string as UTF-8");
+          s[i] = strdup(q);
         }
 
         /* Assign id if not already provided */
@@ -983,21 +976,21 @@ int dlite_swig_set_scalar(void *ptr, DLiteType type, size_t size, obj_t *obj)
       } else if (PyMapping_Check(obj)) {
         char *msg=NULL;
         PyObject *s=NULL, *p=NULL, *o=NULL, *id=NULL;
+        const char *ss, *sp, *so, *sid;
         if (!(s = PyMapping_GetItemString(obj, "s")) ||
             !(p = PyMapping_GetItemString(obj, "p")) ||
             !(o = PyMapping_GetItemString(obj, "o")))
           msg = "Relations must have 's', 'p' and 'o' items";
-        if (!msg && !(PyUnicode_Check(s) && PyUnicode_READY(s) == 0 &&
-                      PyUnicode_Check(p) && PyUnicode_READY(p) == 0 &&
-                      PyUnicode_Check(o) && PyUnicode_READY(o) == 0))
+        if (!msg && !(PyUnicode_Check(s) && (ss = PyUnicode_AsUTF8(s)) &&
+                      PyUnicode_Check(p) && (sp = PyUnicode_AsUTF8(p)) &&
+                      PyUnicode_Check(o) && (so = PyUnicode_AsUTF8(o))))
           msg = "Relation 's', 'p', 'o' items must be strings";
         if (!msg && PyMapping_HasKeyString(obj, "id") &&
             (!(id = PyMapping_GetItemString(obj, "id")) ||
-             !PyUnicode_Check(id) || PyUnicode_READY(id)))
+             !PyUnicode_Check(id) || !(sid = PyUnicode_AsUTF8(id))))
           msg = "If given, relation id must be a string";
         if (!msg)
-          triple_reset(ptr, PyUnicode_DATA(s), PyUnicode_DATA(p),
-                        PyUnicode_DATA(o), (id) ? PyUnicode_DATA(id) : NULL);
+          triple_reset(ptr, ss, sp, so, (id) ? sid : NULL);
         Py_XDECREF(s);
         Py_XDECREF(p);
         Py_XDECREF(o);


Test patch:
diff --git a/bindings/python/tests/CMakeLists.txt b/bindings/python/tests/CMakeLists.txt
index 0420351fc..e0766c9ee 100644
--- a/bindings/python/tests/CMakeLists.txt
+++ b/bindings/python/tests/CMakeLists.txt
@@ -19,6 +19,7 @@ set(tests
   test_rdf
   test_postgresql_write
   test_postgresql_read
+  test_ref_type
   )
 
 foreach(test ${tests})
diff --git a/bindings/python/tests/test_ref_type.json b/bindings/python/tests/test_ref_type.json
new file mode 100644
index 000000000..b5a0daf85
--- /dev/null
+++ b/bindings/python/tests/test_ref_type.json
@@ -0,0 +1,75 @@
+{
+  "http://onto-ns.com/meta/0.1/Top": {
+    "description": "Top-level nested data structure.",
+    "dimensions": {
+      "N": "Number of middle-level structures."
+    },
+    "properties": {
+      "middles": {
+        "type": "ref",
+        "$ref": "http://onto-ns.com/meta/0.1/Middle",
+        "shape": ["N"],
+        "description": "List of middle-level structure references."
+      }
+    }
+  },
+
+  "http://onto-ns.com/meta/0.1/Middle": {
+    "description": "Middle-level nested data structure.",
+    "dimensions": {},
+    "properties": {
+      "name": {
+        "type": "string",
+        "description": "Value of this structure."
+      },
+      "leaf": {
+        "type": "ref",
+        "$ref": "http://onto-ns.com/meta/0.1/Leaf",
+        "description": "Reference to low-level structure."
+      }
+    }
+  },
+
+  "http://onto-ns.com/meta/0.1/Leaf": {
+    "description": "Un-nested data structure.",
+    "dimensions": {},
+    "properties": {
+      "a": {
+        "type": "int",
+        "description": "An integer."
+      },
+      "b": {
+        "type": "bool",
+        "description": "A bool."
+      }
+    }
+  },
+
+  "http://onto-ns.com/meta/0.1/Linked": {
+    "description": "Linked-list.",
+    "dimensions": {},
+    "properties": {
+      "next": {
+        "type": "ref",
+        "$ref": "http://onto-ns.com/meta/0.1/Linked",
+        "description": "Reference next item in the lest."
+      }
+    }
+  },
+
+  "http://onto-ns.com/meta/0.1/Tree": {
+    "description": "Tree-like data structure.",
+    "dimensions": {
+      "N": "Number of leaves structures."
+    },
+    "properties": {
+      "subtree": {
+        "type": "ref",
+        "$ref": "http://onto-ns.com/meta/0.1/Tree",
+        "shape": ["N"],
+        "description": "Reference list of sub-trees."
+      }
+    }
+  }
+
+}
diff --git a/bindings/python/tests/test_ref_type.py b/bindings/python/tests/test_ref_type.py
new file mode 100644
index 000000000..59c155f90
--- /dev/null
+++ b/bindings/python/tests/test_ref_type.py
@@ -0,0 +1,80 @@
+from pathlib import Path
+
+import dlite
+
+
+thisdir = Path(__file__).resolve().parent
+
+dlite.storage_path.append(thisdir / "test_ref_type.json")
+
+Top = dlite.get_instance("http://onto-ns.com/meta/0.1/Top")
+Middle = dlite.get_instance("http://onto-ns.com/meta/0.1/Middle")
+Leaf = dlite.get_instance("http://onto-ns.com/meta/0.1/Leaf")
+Linked = dlite.get_instance("http://onto-ns.com/meta/0.1/Linked")
+Tree = dlite.get_instance("http://onto-ns.com/meta/0.1/Tree")
+
+leaf1 = Leaf(properties={"a": 3, "b": True})
+leaf2 = Leaf(properties={"a": 5, "b": False})
+
+middle1 = Middle(properties={"name": "leaf1", "leaf": leaf1})
+middle2 = Middle(properties={"name": "leaf2", "leaf": leaf2})
+
+top = Top(dimensions=[2], properties={"middles": [middle1, middle2]})
+
+assert top.middles[0].name == "leaf1"
+assert top.middles[0].leaf.a == 3
+assert top.middles[0].leaf.b == True
+assert top.middles[1].name == "leaf2"
+assert top.middles[1].leaf.a == 5
+assert top.middles[1].leaf.b == False
+
+# Test unseting ref
+middle1.leaf = None
+assert middle1.leaf is None
+
+# Test assigning from uuid
+middle1.leaf = leaf1.uuid
+assert middle1.leaf == leaf1
+assert middle1.leaf.a == 3
+
+# Create a linked list
+item0 = Linked()
+item1 = Linked(properties={"next": item0})
+item2 = Linked(properties={"next": item1})
+assert item2.next == item1
+assert item2.next.next == item0
+
+# Convert to a cyclic list
+item0.next = item2
+assert item1.next.next.next == item1  # one cycle
+
+# Create from json
+inst = dlite.Instance.from_json(f"""
+{{
+  "meta": "http://onto-ns.com/meta/0.1/Linked",
+  "dimensions": {{}},
+  "properties": {{
+    "next": "{item0.uuid}"
+  }}
+}}
+""")
+assert inst.next == item0
+assert inst.next.next.next.next == item0
+
+# Create a tree structure
+tree0 = Tree(dimensions=[0])
+tree1 = Tree(dimensions=[2], properties={"subtree": [tree0, tree0]})
+tree2 = Tree(dimensions=[1], properties={"subtree": [tree1]})
+assert tree2.subtree[0].subtree[0] == tree0
+assert tree2.subtree[0].subtree[1] == tree0
+
+# Create a cyclic tree structure
+cyclic = Tree(dimensions=[1])
+cyclic.subtree = [cyclic]
+assert cyclic.subtree[0] == cyclic
+assert cyclic.subtree[0].subtree[0] == cyclic
+assert cyclic.subtree[0].subtree[0].subtree[0] == cyclic
+
+# Instantiate nested from dict
+# For issue #515
+#middle = Middle(properties={"name": "nested", "leaf": {"a": 1, "b": True}})