Repository: SINTEF/dlite (language: python)

Issue:
Calculate hash (as id) from content of instance
Would be very useful/needed in connection distributed hash tables


Discussion / hints:

Fix patch:
diff --git a/src/dlite-entity.c b/src/dlite-entity.c
index d30a91573..5ab28901a 100644
--- a/src/dlite-entity.c
+++ b/src/dlite-entity.c
@@ -10,6 +10,7 @@
 #include "utils/fileutils.h"
 #include "utils/strutils.h"
 #include "utils/infixcalc.h"
+#include "utils/sha3.h"
 
 #include "dlite.h"
 #include "dlite-macros.h"
@@ -914,18 +915,6 @@ DLiteInstance *_instance_load_casted(const DLiteStorage *s, const char *id,
         FAIL2("metadata %s loaded from %s has no name, version and namespace",
              id, s->location);
       }
-    //} else {
-    //  char **dataname;
-    //  ErrTry:
-    //    dataname = dlite_instance_get_property(inst, "dataname");
-    //    break;
-    //  ErrOther:
-    //    break;
-    //  ErrEnd;
-    //  if (dataname) {
-    //    inst->uri = strdup(*dataname);
-    //    dlite_get_uuid(inst->uuid, inst->uri);
-    //  }
     }
   }
 
@@ -1947,6 +1936,51 @@ int dlite_instance_assign_casted_property_by_index(const DLiteInstance *inst,
 }
 
 
+/*
+  Calculates a hash of instance `inst`.  The calculated hash is stored
+  in `hash`, where `hashsize` is the size of `hash` in bytes.  It should
+  be 32, 48 or 64.
+
+  Return non-zero on error.
+ */
+int dlite_instance_get_hash(const DLiteInstance *inst,
+                            unsigned char *hash, int hashsize)
+{
+  size_t i;
+  sha3_context c;
+  const unsigned char *buf;
+  unsigned bitsize = hashsize * 8;
+  int retval = 0;
+  sha3_Init(&c, bitsize);
+  sha3_SetFlags(&c, SHA3_FLAGS_KECCAK);
+
+  sha3_Update(&c, inst->uuid, DLITE_UUID_LENGTH);
+  sha3_Update(&c, inst->meta->uri, strlen(inst->meta->uri));
+  for (i=0; i<DLITE_NDIM(inst); i++) {
+    size_t n = DLITE_DIM(inst, i);
+    sha3_Update(&c, &n, sizeof(size_t));
+  }
+  for (i=0; i<DLITE_NPROP(inst); i++) {
+    void *ptr = dlite_instance_get_property_by_index(inst, i);
+    DLiteProperty *p = DLITE_PROP_DESCR(inst, i);
+    size_t j, len = 1;
+    for (j=0; (int)j<p->ndims; j++) len *= DLITE_PROP_DIM(inst, i, j);
+    if (dlite_type_is_allocated(p->type)) {
+      char *v = ptr;
+      for (j=0; j<len; j++, v+=p->size)
+        if ((retval = dlite_type_update_sha3(&c, v, p->type, p->size)))
+          break;
+    } else {
+      sha3_Update(&c, ptr, len*p->size);
+    }
+  }
+
+  buf = sha3_Finalize(&c);
+  memcpy(hash, buf, hashsize);
+  return retval;
+}
+
+
 
 /********************************************************************
  *  Metadata
diff --git a/src/dlite-entity.h b/src/dlite-entity.h
index 7b2e33d7a..8930ab70d 100644
--- a/src/dlite-entity.h
+++ b/src/dlite-entity.h
@@ -964,6 +964,16 @@ int dlite_instance_assign_casted_property_by_index(const DLiteInstance *inst,
                                                    const void *src,
                                                    DLiteTypeCast castfun);
 
+/**
+  Calculates a hash of instance `inst`.  The calculated hash is stored
+  in `hash`, where `hashsize` is the size of `hash` in bytes.  It should
+  be 32, 48 or 64.
+
+  Return non-zero on error.
+ */
+int dlite_instance_get_hash(const DLiteInstance *inst,
+                            unsigned char *hash, int hashsize);
+
 
 /** @} */
 /* ================================================================= */
diff --git a/src/dlite-type.c b/src/dlite-type.c
index 5f1557cbc..012ebfd6a 100644
--- a/src/dlite-type.c
+++ b/src/dlite-type.c
@@ -1032,7 +1032,11 @@ int dlite_type_scan(const char *src, int len, void *p, DLiteType dtype,
 
       if (len < 0) len = strlen(src);
       jsmn_init(&parser);
-      if ((r = jsmn_parse(&parser, src, len, tokens, MAX_PROPERTY_TOKENS)) < 0)
+      r = jsmn_parse(&parser, src, len, tokens, MAX_PROPERTY_TOKENS);
+      if (r == JSMN_ERROR_NOMEM)
+        return err(-1, "too many dimensions.  Increase MAX_PROPERTY_TOKENS "
+                   "in dlite-type.c and recompile.");
+      else if (r < 0)
         return err(-1, "cannot parse property: %s: '%s'",
                    jsmn_strerror(r), src);
       if (tokens->type != JSMN_OBJECT)
@@ -1120,6 +1124,64 @@ int dlite_type_scan(const char *src, int len, void *p, DLiteType dtype,
 }
 
 
+/*
+  Update sha3 hash context `c` from data pointed to by `ptr`.
+  The data is described by `dtype` and `size`.
+
+  Returns non-zero on error.
+ */
+int dlite_type_update_sha3(sha3_context *c, const void *ptr,
+                           DLiteType dtype, size_t size)
+{
+  switch (dtype) {
+
+  case dliteStringPtr:
+    {
+      char *s = *((char **)ptr);
+      if (s) sha3_Update(c, s, strlen(s));
+    }
+    break;
+
+  case dliteDimension:
+    {
+      const DLiteDimension *d = ptr;
+      sha3_Update(c, d->name, strlen(d->name));
+      //if (d->description) sha3_Update(c, d->description);
+    }
+    break;
+
+  case dliteProperty:
+    {
+      int i;
+      const DLiteProperty *p = ptr;
+      sha3_Update(c, p->name, strlen(p->name));
+      sha3_Update(c, &p->type, sizeof(DLiteType));
+      sha3_Update(c, &p->size, sizeof(size_t));
+      sha3_Update(c, &p->ndims, sizeof(int));
+      for (i=0; i<p->ndims; i++)
+        sha3_Update(c, p->dims[i], strlen(p->dims[i]));
+      if (p->unit) sha3_Update(c, p->unit, strlen(p->unit));
+      //if (p->description)
+      //  sha3_Update(c, p->description, strlen(p->description));
+    }
+    break;
+
+  case dliteRelation:
+    {
+      const DLiteRelation *rel = ptr;
+      sha3_Update(c, rel->s, strlen(rel->s));
+      sha3_Update(c, rel->p, strlen(rel->p));
+      sha3_Update(c, rel->o, strlen(rel->o));
+    }
+    break;
+
+  default:
+    sha3_Update(c, ptr, size);
+    break;
+  }
+  return 0;
+}
+
 
 /*
   Returns the struct alignment of the given type or 0 on error.
diff --git a/src/dlite-type.h b/src/dlite-type.h
index 9b609548e..f0c0ab839 100644
--- a/src/dlite-type.h
+++ b/src/dlite-type.h
@@ -69,6 +69,7 @@
 #include <stdlib.h>
 
 #include "utils/boolean.h"
+#include "utils/sha3.h"
 #include "triplestore.h"
 
 
@@ -264,6 +265,16 @@ int dlite_type_aprint(char **dest, size_t *n, size_t pos, const void *p,
 int dlite_type_scan(const char *src, int len, void *p, DLiteType dtype,
                     size_t size, DLiteTypeFlag flags);
 
+/**
+  Update sha3 hash context `c` from data pointed to by `ptr`.
+  The data is described by `dtype` and `size`.
+
+  Returns non-zero on error.
+ */
+int dlite_type_update_sha3(sha3_context *c, const void *ptr,
+                           DLiteType dtype, size_t size);
+
+
 /**
   Returns the struct alignment of the given type or 0 on error.
  */


Test patch:
diff --git a/src/tests/test_entity.c b/src/tests/test_entity.c
index ea82e9099..ec58c9b62 100644
--- a/src/tests/test_entity.c
+++ b/src/tests/test_entity.c
@@ -6,6 +6,7 @@
 #include "minunit/minunit.h"
 #include "utils/integers.h"
 #include "utils/boolean.h"
+#include "utils/strutils.h"
 #include "dlite.h"
 #include "dlite-entity.h"
 #include "dlite-storage.h"
@@ -330,6 +331,46 @@ MU_TEST(test_instance_snprint)
 }
 
 
+/* Write hex encoded hash to string `s`, which must  be at least 65 bytes. */
+static char *gethash(char *s, DLiteInstance *inst)
+{
+  unsigned char buf[32];
+  size_t hashsize = sizeof(buf);
+  dlite_instance_get_hash(inst, buf, hashsize);
+  if (strhex_encode(s, 65, buf, hashsize) < 0) return NULL;
+  return s;
+}
+
+
+MU_TEST(test_instance_get_hash)
+{
+  char *hash;
+  char s[65];
+  DLiteInstance *inst = dlite_instance_get("mydata");
+  mu_check(inst);
+
+  hash = "685c5cf484305b7db0650730e8e9c00bbf18981ffac4e5fc2d8a73250d4b2cef";
+  mu_assert_string_eq(hash, gethash(s, inst));
+
+  // metadata
+  hash = "d4d51d72a4cd9ef7fd8a6071e28cc4309719763273c1b27f6b4cd3409f164466";
+  mu_assert_string_eq(hash, gethash(s, (DLiteInstance *)inst->meta));
+
+  // metadata schema
+  hash = "d6d28a55e987f90e08edb553aa5ae811b7bb9459294c36c6ea8ea3899ba9b22a";
+  mu_assert_string_eq(hash, gethash(s, (DLiteInstance *)inst->meta->meta));
+
+  // basic metadata schema
+  hash = "a7d885be5227c2e4ac3b3686c5a603566d70e1563ee221d4cdb24f53569c1ce5";
+  mu_assert_string_eq(hash, gethash(s, (DLiteInstance *)inst->meta->meta->meta));
+
+  // basic metadata schema
+  mu_assert_string_eq(hash, gethash(s, (DLiteInstance *)inst->meta->meta->meta->meta));
+
+  dlite_instance_decref(inst);
+}
+
+
 MU_TEST(test_meta_save)
 {
   DLiteStorage *s;
@@ -410,6 +451,7 @@ MU_TEST_SUITE(test_suite)
   MU_RUN_TEST(test_instance_json);
   MU_RUN_TEST(test_instance_load_url);
   MU_RUN_TEST(test_instance_snprint);
+  MU_RUN_TEST(test_instance_get_hash);
 
   MU_RUN_TEST(test_meta_save);
   MU_RUN_TEST(test_meta_load);
diff --git a/src/tests/test_type.c b/src/tests/test_type.c
index aac4a7b95..c40fbeb0c 100644
--- a/src/tests/test_type.c
+++ b/src/tests/test_type.c
@@ -8,6 +8,7 @@
 #include "minunit/minunit.h"
 #include "utils/integers.h"
 #include "utils/boolean.h"
+#include "utils/strutils.h"
 #include "utils/err.h"
 #include "dlite.h"
 
@@ -380,6 +381,75 @@ MU_TEST(test_scan)
 }
 
 
+/* Write hex encoded hash to string `s`, which must  be at least 65 bytes. */
+static char *gethash(char *s, const void *ptr, DLiteType dtype, size_t size)
+{
+  sha3_context c;
+  int hashsize = 32;
+  int bitsize = 8*hashsize;
+  const unsigned char *buf;
+  sha3_Init(&c, bitsize);
+  sha3_SetFlags(&c, SHA3_FLAGS_KECCAK);
+  if (dlite_type_update_sha3(&c, ptr, dtype, size)) return NULL;
+  buf = sha3_Finalize(&c);
+  if (strhex_encode(s, 65, buf, hashsize) < 0) return NULL;
+  return s;
+}
+
+MU_TEST(test_update_sha3)
+{
+  char s[65];
+  const char *hash;
+
+  int32_t i1 = 42;
+  hash = "298c8f103b5a4112a1ab1da335986cfc363f068fcd72c0393382d02a71faa24a";
+  mu_assert_string_eq(hash, gethash(s, &i1, dliteInt, sizeof(i1)));
+
+  int32_t i2 = 43;
+  hash = "b81c9c72c6322c9aa98c64259488c6a7d27d3638aa329cf272a2c5d1c5637cd6";
+  mu_assert_string_eq(hash, gethash(s, &i2, dliteInt, sizeof(i2)));
+
+  bool b1 = 0;
+  hash = "bc36789e7a1e281436464229828f817d6612f7b477d66591ff96a9e064bcc98a";
+  mu_assert_string_eq(hash, gethash(s, &b1, dliteBool, sizeof(b1)));
+
+  bool b2 = 1;
+  hash = "5fe7f977e71dba2ea1a68e21057beebb9be2ac30c6410aa38d4f3fbe41dcffd2";
+  mu_assert_string_eq(hash, gethash(s, &b2, dliteBool, sizeof(b2)));
+
+  bool b3 = -1;
+  hash = "5fe7f977e71dba2ea1a68e21057beebb9be2ac30c6410aa38d4f3fbe41dcffd2";
+  mu_assert_string_eq(hash, gethash(s, &b3, dliteBool, sizeof(b3)));
+
+  char s1[] = "string1";
+  hash = "22bceddf404e46d56d0d3770553d3b88745675ea98806dd2adedbad333ff2e9c";
+  mu_assert_string_eq(hash, gethash(s, s1, dliteFixString, strlen(s1)));
+
+  char *s2 = s1;
+  hash = "22bceddf404e46d56d0d3770553d3b88745675ea98806dd2adedbad333ff2e9c";
+  mu_assert_string_eq(hash, gethash(s, &s2, dliteStringPtr, sizeof(s2)));
+
+  DLiteDimension d1 = {"dimname", "dimdescr"};
+  hash = "fc28849f70bcc72785d7f8d89ccbd9b1ffb71674bdd4d8dd327c78c7052c3bdb";
+  mu_assert_string_eq(hash, gethash(s, &d1, dliteDimension, sizeof(d1)));
+
+  DLiteDimension d2 = {"dimname", "???"};  // hash independent of description
+  mu_assert_string_eq(hash, gethash(s, &d2, dliteDimension, sizeof(d2)));
+
+  char *dims[] = {"dim1", "dim2"};
+  DLiteProperty p = {"propname", dliteStringPtr, sizeof(char *), 2, dims,
+    "m/s", NULL};
+  hash = "f52af54cb773ec8a7499ae44f3499c58fba40203d870c0c52842d10cbbcdc13a";
+  mu_assert_string_eq(hash, gethash(s, &p, dliteProperty, sizeof(p)));
+
+  p.description = "Some description...";  // hash independent of description
+  mu_assert_string_eq(hash, gethash(s, &p, dliteProperty, sizeof(p)));
+
+  p.unit = "m";
+  hash = "f92e79baa064d9eefc1e0790bf6332678a792a5e1cb28d94a237abee42de1505";
+  mu_assert_string_eq(hash, gethash(s, &p, dliteProperty, sizeof(p)));
+}
+
 
 MU_TEST(test_get_alignment)
 {
@@ -552,6 +622,7 @@ MU_TEST_SUITE(test_suite)
   MU_RUN_TEST(test_clear);
   MU_RUN_TEST(test_print);
   MU_RUN_TEST(test_scan);
+  MU_RUN_TEST(test_update_sha3);
   MU_RUN_TEST(test_get_alignment);
   MU_RUN_TEST(test_padding_at);
   MU_RUN_TEST(test_get_member_offset);