Repository: Wikidata/Wikidata-Toolkit
Pull request #473 (2019-12-24 16:50:18)

JSON deserialization helpers and fixes
This adds a deserialization helper for #471, and fixes deserialization of forms and senses (#472).
JSON payloads retrieved from Wikidata are used for testing.


Patch:
diff --git a/pom.xml b/pom.xml
index ff855aba..0bc11fd8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,6 +74,7 @@
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<apacheCommonsCompressVersion>1.19</apacheCommonsCompressVersion>
 		<apacheCommonsLangVersion>3.9</apacheCommonsLangVersion>
+		<apacheCommonsIOVersion>1.3.2</apacheCommonsIOVersion>
 		<apacheHttpVersion>4.5.10</apacheHttpVersion>
 		<commonsCliVersion>1.4</commonsCliVersion>
 		<ini4JVersion>0.5.4</ini4JVersion>
diff --git a/wdtk-datamodel/pom.xml b/wdtk-datamodel/pom.xml
index b185e37b..dba24fd7 100644
--- a/wdtk-datamodel/pom.xml
+++ b/wdtk-datamodel/pom.xml
@@ -46,6 +46,12 @@
 			<artifactId>threeten-extra</artifactId>
 			<version>${threetenVersion}</version>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-io</artifactId>
+			<version>${apacheCommonsIOVersion}</version>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 	
 	<build>
diff --git a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/JsonDeserializer.java b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/JsonDeserializer.java
new file mode 100644
index 00000000..878cc7b1
--- /dev/null
+++ b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/JsonDeserializer.java
@@ -0,0 +1,97 @@
+package org.wikidata.wdtk.datamodel.helpers;
+
+import java.io.IOException;
+
+import org.wikidata.wdtk.datamodel.implementation.EntityDocumentImpl;
+import org.wikidata.wdtk.datamodel.implementation.ItemDocumentImpl;
+import org.wikidata.wdtk.datamodel.implementation.LexemeDocumentImpl;
+import org.wikidata.wdtk.datamodel.implementation.MediaInfoDocumentImpl;
+import org.wikidata.wdtk.datamodel.implementation.PropertyDocumentImpl;
+import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
+import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
+import org.wikidata.wdtk.datamodel.interfaces.LexemeDocument;
+import org.wikidata.wdtk.datamodel.interfaces.MediaInfoDocument;
+import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+
+/**
+ * Helper to deserialize datamodel objects from their
+ * JSON representation.
+ * 
+ * We accept empty arrays as empty maps since there has
+ * been a confusion in the past between the two:
+ * https://phabricator.wikimedia.org/T138104
+ * 
+ * @author Antonin Delpeuch
+ */
+public class JsonDeserializer {
+	
+	protected DatamodelMapper mapper;
+	
+	/**
+	 * Constructs a new JSON deserializer for the 
+	 * designated site.
+	 * 
+	 * @param siteIri
+	 * 		Root IRI of the site to deserialize for
+	 */
+	public JsonDeserializer(String siteIri) {
+		mapper = new DatamodelMapper(siteIri);
+	}
+	
+	/**
+	 * Deserializes a JSON string into an {@class ItemDocument}.
+	 * @throws IOException 
+			if the JSON payload is invalid
+	 */
+	public ItemDocument deserializeItemDocument(String json) throws IOException {
+		return mapper.readerFor(ItemDocumentImpl.class)
+				.with(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT)
+				.readValue(json);
+	}
+	
+	/**
+	 * Deserializes a JSON string into a {@class PropertyDocument}.
+	 * @throws IOException 
+			if the JSON payload is invalid
+	 */
+	public PropertyDocument deserializePropertyDocument(String json) throws IOException {
+		return mapper.readerFor(PropertyDocumentImpl.class)
+				.with(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT)
+				.readValue(json);
+	}
+
+	/**
+	 * Deserializes a JSON string into a {@class LexemeDocument}.
+	 * @throws IOException 
+			if the JSON payload is invalid
+	 */
+	public LexemeDocument deserializeLexemeDocument(String json) throws IOException {
+		return mapper.readerFor(LexemeDocumentImpl.class)
+				.with(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT)
+				.readValue(json);
+	}
+	
+	/**
+	 * Deserializes a JSON string into a {@class MediaInfoDocument}.
+	 * @throws IOException 
+			if the JSON payload is invalid
+	 */
+	public MediaInfoDocument deserializeMediaInfoDocument(String json) throws IOException {
+		return mapper.readerFor(MediaInfoDocumentImpl.class)
+				.with(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT)
+				.readValue(json);
+	}
+	
+	/**
+	 * Deserializes a JSON string into a {@class EntityDocument}.
+	 * @throws IOException 
+			if the JSON payload is invalid
+	 */
+	public EntityDocument deserializeEntityDocument(String json) throws IOException {
+		return mapper.readerFor(EntityDocumentImpl.class)
+				.with(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT)
+				.readValue(json);
+	}
+}
diff --git a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/JsonSerializer.java b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/JsonSerializer.java
index 2259c7e8..ed7c4d4e 100644
--- a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/JsonSerializer.java
+++ b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/JsonSerializer.java
@@ -26,7 +26,18 @@ import java.nio.charset.StandardCharsets;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.wikidata.wdtk.datamodel.interfaces.*;
+import org.wikidata.wdtk.datamodel.implementation.EntityDocumentImpl;
+import org.wikidata.wdtk.datamodel.implementation.ItemDocumentImpl;
+import org.wikidata.wdtk.datamodel.implementation.LexemeDocumentImpl;
+import org.wikidata.wdtk.datamodel.implementation.MediaInfoDocumentImpl;
+import org.wikidata.wdtk.datamodel.implementation.PropertyDocumentImpl;
+import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
+import org.wikidata.wdtk.datamodel.interfaces.EntityDocumentDumpProcessor;
+import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
+import org.wikidata.wdtk.datamodel.interfaces.LexemeDocument;
+import org.wikidata.wdtk.datamodel.interfaces.MediaInfoDocument;
+import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
+import org.wikidata.wdtk.datamodel.interfaces.Statement;
 
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -232,5 +243,4 @@ public class JsonSerializer implements EntityDocumentDumpProcessor {
 			return null;
 		}
 	}
-
 }
diff --git a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/FormDocumentImpl.java b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/FormDocumentImpl.java
index 685d7f4c..07cda9bd 100644
--- a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/FormDocumentImpl.java
+++ b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/FormDocumentImpl.java
@@ -37,6 +37,7 @@ import java.util.*;
  * @author Thomas Pellissier Tanon
  */
 @JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
 public class FormDocumentImpl extends StatementDocumentImpl implements FormDocument {
 
 	private final List<ItemIdValue> grammaticalFeatures;
diff --git a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/SenseDocumentImpl.java b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/SenseDocumentImpl.java
index 79f8308a..52f80384 100644
--- a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/SenseDocumentImpl.java
+++ b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/SenseDocumentImpl.java
@@ -40,6 +40,7 @@ import java.util.Set;
  * @author Thomas Pellissier Tanon
  */
 @JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeInfo(use = JsonTypeInfo.Id.NONE)
 public class SenseDocumentImpl extends StatementDocumentImpl implements SenseDocument {
 
 	private final Map<String,MonolingualTextValue> glosses;


Test patch:
diff --git a/wdtk-datamodel/src/test/java/org/wikidata/wdtk/datamodel/helpers/JsonDeserializerTest.java b/wdtk-datamodel/src/test/java/org/wikidata/wdtk/datamodel/helpers/JsonDeserializerTest.java
new file mode 100644
index 00000000..ea516f15
--- /dev/null
+++ b/wdtk-datamodel/src/test/java/org/wikidata/wdtk/datamodel/helpers/JsonDeserializerTest.java
@@ -0,0 +1,58 @@
+package org.wikidata.wdtk.datamodel.helpers;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.wikidata.wdtk.datamodel.interfaces.EntityDocument;
+import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
+import org.wikidata.wdtk.datamodel.interfaces.LexemeDocument;
+import org.wikidata.wdtk.datamodel.interfaces.MediaInfoDocument;
+import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
+
+
+public class JsonDeserializerTest {
+	
+	public JsonDeserializer SUT = new JsonDeserializer(Datamodel.SITE_WIKIDATA);
+	public JsonDeserializer SUTcommons = new JsonDeserializer(Datamodel.SITE_WIKIMEDIA_COMMONS);
+	
+	protected String loadJson(String filename) throws IOException {
+		InputStream stream = JsonDeserializerTest.class.getClassLoader()
+				.getResourceAsStream("JsonDeserializer/"+filename);
+		return IOUtils.toString(stream);
+	}
+
+	@Test
+	public void testLoadItemDocument() throws IOException {
+		ItemDocument doc = SUT.deserializeItemDocument(loadJson("item.json"));
+		Assert.assertEquals(doc.getEntityId(), Datamodel.makeWikidataItemIdValue("Q34987"));
+	}
+	
+	@Test
+	public void testLoadPropertyDocument() throws IOException {
+		PropertyDocument doc = SUT.deserializePropertyDocument(loadJson("property.json"));
+		Assert.assertEquals(doc.getEntityId(), Datamodel.makeWikidataPropertyIdValue("P3467"));
+	}
+	
+	@Test
+	public void testLoadLexemeDocument() throws IOException {
+		LexemeDocument doc = SUT.deserializeLexemeDocument(loadJson("lexeme.json"));
+		Assert.assertEquals(doc.getEntityId(), Datamodel.makeWikidataLexemeIdValue("L3872"));
+		Assert.assertEquals(doc.getForm(Datamodel.makeWikidataFormIdValue("L3872-F2")).getStatementGroups(), Collections.emptyList());
+	}
+	
+	@Test
+	public void testLoadMediaInfoDocument() throws IOException {
+		MediaInfoDocument doc = SUTcommons.deserializeMediaInfoDocument(loadJson("mediainfo.json"));
+		Assert.assertEquals(doc.getEntityId(), Datamodel.makeWikimediaCommonsMediaInfoIdValue("M74698470"));
+	}
+	
+	@Test
+	public void testDeserializeEntityDocument() throws IOException {
+		EntityDocument doc = SUT.deserializeEntityDocument(loadJson("property.json"));
+		Assert.assertEquals(doc.getEntityId(), Datamodel.makeWikidataPropertyIdValue("P3467"));
+	}
+}
diff --git a/wdtk-datamodel/src/test/resources/JsonDeserializer/item.json b/wdtk-datamodel/src/test/resources/JsonDeserializer/item.json
new file mode 100644
index 00000000..157e921a
--- /dev/null
+++ b/wdtk-datamodel/src/test/resources/JsonDeserializer/item.json
@@ -0,0 +1,755 @@
+{
+  "pageid": 37687,
+  "ns": 0,
+  "title": "Q34987",
+  "lastrevid": 1062625956,
+  "modified": "2019-11-26T12:42:18Z",
+  "type": "item",
+  "id": "Q34987",
+  "labels": {
+    "sw": {
+      "language": "sw",
+      "value": "Kibile"
+    },
+    "de": {
+      "language": "de",
+      "value": "Bile"
+    },
+    "pms": {
+      "language": "pms",
+      "value": "Lenga Bile"
+    },
+    "en": {
+      "language": "en",
+      "value": "Bile"
+    },
+    "hr": {
+      "language": "hr",
+      "value": "Bile jezik"
+    },
+    "ru": {
+      "language": "ru",
+      "value": "Биле"
+    },
+    "en-gb": {
+      "language": "en-gb",
+      "value": "Bile"
+    },
+    "ce": {
+      "language": "ce",
+      "value": "Биле"
+    }
+  },
+  "descriptions": {
+    "de": {
+      "language": "de",
+      "value": "Sprache"
+    },
+    "en": {
+      "language": "en",
+      "value": "language"
+    },
+    "he": {
+      "language": "he",
+      "value": "שפה"
+    },
+    "br": {
+      "language": "br",
+      "value": "yezh"
+    },
+    "fr": {
+      "language": "fr",
+      "value": "langue"
+    },
+    "es": {
+      "language": "es",
+      "value": "lengua"
+    },
+    "it": {
+      "language": "it",
+      "value": "lingua"
+    },
+    "nl": {
+      "language": "nl",
+      "value": "taal"
+    },
+    "la": {
+      "language": "la",
+      "value": "lingua"
+    },
+    "cy": {
+      "language": "cy",
+      "value": "iaith"
+    }
+  },
+  "aliases": {
+    "en": [
+      {
+        "language": "en",
+        "value": "Bile language"
+      }
+    ]
+  },
+  "claims": {
+    "P220": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P220",
+          "hash": "77ed58adb4323efb034906aeb3b7d8cc8936c56c",
+          "datavalue": {
+            "value": "bil",
+            "type": "string"
+          },
+          "datatype": "external-id"
+        },
+        "type": "statement",
+        "id": "Q34987$A5EE01BE-0FD2-478C-9405-5ECF86B0DF9B",
+        "rank": "normal",
+        "references": [
+          {
+            "hash": "56b370ad342ff9d47a4119b3f53f894995cea4b7",
+            "snaks": {
+              "P248": [
+                {
+                  "snaktype": "value",
+                  "property": "P248",
+                  "hash": "27fbd18e13a0a2c1e2ae172bfbfc66577ffb38a2",
+                  "datavalue": {
+                    "value": {
+                      "entity-type": "item",
+                      "numeric-id": 75488338,
+                      "id": "Q75488338"
+                    },
+                    "type": "wikibase-entityid"
+                  },
+                  "datatype": "wikibase-item"
+                }
+              ],
+              "P854": [
+                {
+                  "snaktype": "value",
+                  "property": "P854",
+                  "hash": "68a06f5f5ab362597f2d12e45856619795420364",
+                  "datavalue": {
+                    "value": "https://op.europa.eu/web/eu-vocabularies/at-dataset/-/resource/dataset/language",
+                    "type": "string"
+                  },
+                  "datatype": "url"
+                }
+              ]
+            },
+            "snaks-order": [
+              "P248",
+              "P854"
+            ]
+          },
+          {
+            "hash": "fa278ebfc458360e5aed63d5058cca83c46134f1",
+            "snaks": {
+              "P143": [
+                {
+                  "snaktype": "value",
+                  "property": "P143",
+                  "hash": "e4f6d9441d0600513c4533c672b5ab472dc73694",
+                  "datavalue": {
+                    "value": {
+                      "entity-type": "item",
+                      "numeric-id": 328,
+                      "id": "Q328"
+                    },
+                    "type": "wikibase-entityid"
+                  },
+                  "datatype": "wikibase-item"
+                }
+              ]
+            },
+            "snaks-order": [
+              "P143"
+            ]
+          }
+        ]
+      }
+    ],
+    "P31": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P31",
+          "hash": "ac8ce0eb2aee500f93443c85f173bdbfece859a4",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 34770,
+              "id": "Q34770"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "Q34987$8B2CFC26-08CD-4366-8ABC-FD28B39392C2",
+        "rank": "normal",
+        "references": [
+          {
+            "hash": "b096ce736246deae93782819465859680c66015c",
+            "snaks": {
+              "P248": [
+                {
+                  "snaktype": "value",
+                  "property": "P248",
+                  "hash": "375610daaf9b20dda81cd83553d79636108b64cb",
+                  "datavalue": {
+                    "value": {
+                      "entity-type": "item",
+                      "numeric-id": 14790,
+                      "id": "Q14790"
+                    },
+                    "type": "wikibase-entityid"
+                  },
+                  "datatype": "wikibase-item"
+                }
+              ]
+            },
+            "snaks-order": [
+              "P248"
+            ]
+          }
+        ]
+      },
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P31",
+          "hash": "61edb75e9440807d73635f592270b56ef4b18020",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 1288568,
+              "id": "Q1288568"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "Q34987$E9A36CAC-E888-4F3C-9298-83468DFE4E8F",
+        "rank": "normal",
+        "references": [
+          {
+            "hash": "ca9fe08a869c1c95f210c9ac9e0fd47b086a8379",
+            "snaks": {
+              "P854": [
+                {
+                  "snaktype": "value",
+                  "property": "P854",
+                  "hash": "fed318b7dbb903a7b459b51df9c56dde40e8dce7",
+                  "datavalue": {
+                    "value": "https://iso639-3.sil.org/code/bil",
+                    "type": "string"
+                  },
+                  "datatype": "url"
+                }
+              ]
+            },
+            "snaks-order": [
+              "P854"
+            ]
+          }
+        ]
+      }
+    ],
+    "P646": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P646",
+          "hash": "3a20650bf2b9eeac4cf2f81e9bccfbd9698c5c5f",
+          "datavalue": {
+            "value": "/m/0h94lsr",
+            "type": "string"
+          },
+          "datatype": "external-id"
+        },
+        "type": "statement",
+        "id": "Q34987$4E0156BD-51F3-442C-A7D3-D393A8620DD0",
+        "rank": "normal",
+        "references": [
+          {
+            "hash": "2b00cb481cddcac7623114367489b5c194901c4a",
+            "snaks": {
+              "P248": [
+                {
+                  "snaktype": "value",
+                  "property": "P248",
+                  "hash": "a94b740202b097dd33355e0e6c00e54b9395e5e0",
+                  "datavalue": {
+                    "value": {
+                      "entity-type": "item",
+                      "numeric-id": 15241312,
+                      "id": "Q15241312"
+                    },
+                    "type": "wikibase-entityid"
+                  },
+                  "datatype": "wikibase-item"
+                }
+              ],
+              "P577": [
+                {
+                  "snaktype": "value",
+                  "property": "P577",
+                  "hash": "fde79ecb015112d2f29229ccc1ec514ed3e71fa2",
+                  "datavalue": {
+                    "value": {
+                      "time": "+2013-10-28T00:00:00Z",
+                      "timezone": 0,
+                      "before": 0,
+                      "after": 0,
+                      "precision": 11,
+                      "calendarmodel": "http://www.wikidata.org/entity/Q1985727"
+                    },
+                    "type": "time"
+                  },
+                  "datatype": "time"
+                }
+              ]
+            },
+            "snaks-order": [
+              "P248",
+              "P577"
+            ]
+          }
+        ]
+      }
+    ],
+    "P1394": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P1394",
+          "hash": "110e1758d4edc808f5618943d8f7ea2eb492bb39",
+          "datavalue": {
+            "value": "bile1244",
+            "type": "string"
+          },
+          "datatype": "external-id"
+        },
+        "type": "statement",
+        "id": "Q34987$9353E43A-0D74-4791-812D-087D4087ADAB",
+        "rank": "normal",
+        "references": [
+          {
+            "hash": "fa278ebfc458360e5aed63d5058cca83c46134f1",
+            "snaks": {
+              "P143": [
+                {
+                  "snaktype": "value",
+                  "property": "P143",
+                  "hash": "e4f6d9441d0600513c4533c672b5ab472dc73694",
+                  "datavalue": {
+                    "value": {
+                      "entity-type": "item",
+                      "numeric-id": 328,
+                      "id": "Q328"
+                    },
+                    "type": "wikibase-entityid"
+                  },
+                  "datatype": "wikibase-item"
+                }
+              ]
+            },
+            "snaks-order": [
+              "P143"
+            ]
+          }
+        ]
+      }
+    ],
+    "P1014": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P1014",
+          "hash": "e4e5eea85ea1a5e5004c5571d77682bdbd53b06d",
+          "datavalue": {
+            "value": "300264027",
+            "type": "string"
+          },
+          "datatype": "external-id"
+        },
+        "type": "statement",
+        "id": "Q34987$F921B970-E45E-4D03-A201-1DAAB8889C02",
+        "rank": "normal"
+      }
+    ],
+    "P305": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P305",
+          "hash": "f76b15a9f6825d800d7ba1244f50303c27d6ae41",
+          "datavalue": {
+            "value": "bil",
+            "type": "string"
+          },
+          "datatype": "external-id"
+        },
+        "type": "statement",
+        "qualifiers": {
+          "P580": [
+            {
+              "snaktype": "value",
+              "property": "P580",
+              "hash": "47d0d9e39591d66e42099639ca9b282d03d1b039",
+              "datavalue": {
+                "value": {
+                  "time": "+2009-07-29T00:00:00Z",
+                  "timezone": 0,
+                  "before": 0,
+                  "after": 0,
+                  "precision": 11,
+                  "calendarmodel": "http://www.wikidata.org/entity/Q1985727"
+                },
+                "type": "time"
+              },
+              "datatype": "time"
+            }
+          ]
+        },
+        "qualifiers-order": [
+          "P580"
+        ],
+        "id": "Q34987$98A36E34-18B7-4FDF-B6F9-AE1A6065ABAE",
+        "rank": "normal",
+        "references": [
+          {
+            "hash": "4aa3c46831638dff4cf3c42ae0f0acf3b3e1234a",
+            "snaks": {
+              "P248": [
+                {
+                  "snaktype": "value",
+                  "property": "P248",
+                  "hash": "14411bc8b193205898cff7811c162db62ecec994",
+                  "datavalue": {
+                    "value": {
+                      "entity-type": "item",
+                      "numeric-id": 57271947,
+                      "id": "Q57271947"
+                    },
+                    "type": "wikibase-entityid"
+                  },
+                  "datatype": "wikibase-item"
+                }
+              ],
+              "P813": [
+                {
+                  "snaktype": "value",
+                  "property": "P813",
+                  "hash": "f0f70081902969e9832f43d66e5825e99355508e",
+                  "datavalue": {
+                    "value": {
+                      "time": "+2019-02-08T00:00:00Z",
+                      "timezone": 0,
+                      "before": 0,
+                      "after": 0,
+                      "precision": 11,
+                      "calendarmodel": "http://www.wikidata.org/entity/Q1985727"
+                    },
+                    "type": "time"
+                  },
+                  "datatype": "time"
+                }
+              ],
+              "P577": [
+                {
+                  "snaktype": "value",
+                  "property": "P577",
+                  "hash": "11cec956c1dfec04779f742dbf48c43a5cd07719",
+                  "datavalue": {
+                    "value": {
+                      "time": "+2009-07-29T00:00:00Z",
+                      "timezone": 0,
+                      "before": 0,
+                      "after": 0,
+                      "precision": 11,
+                      "calendarmodel": "http://www.wikidata.org/entity/Q1985727"
+                    },
+                    "type": "time"
+                  },
+                  "datatype": "time"
+                }
+              ],
+              "P1476": [
+                {
+                  "snaktype": "value",
+                  "property": "P1476",
+                  "hash": "61c3b0f256fd7c592d87366c8539a8a4e53b3b14",
+                  "datavalue": {
+                    "value": {
+                      "text": "Bile",
+                      "language": "en"
+                    },
+                    "type": "monolingualtext"
+                  },
+                  "datatype": "monolingualtext"
+                }
+              ]
+            },
+            "snaks-order": [
+              "P248",
+              "P813",
+              "P577",
+              "P1476"
+            ]
+          }
+        ]
+      }
+    ],
+    "P1627": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P1627",
+          "hash": "081219177c4d274bfbe05fd6ccb06ac375f1a06d",
+          "datavalue": {
+            "value": "bil",
+            "type": "string"
+          },
+          "datatype": "external-id"
+        },
+        "type": "statement",
+        "id": "Q34987$862FDABC-75DB-4ECD-9ABB-3CE8FDA59CA9",
+        "rank": "normal"
+      }
+    ],
+    "P3823": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P3823",
+          "hash": "5e5fdd6a244fe705c817046992d272db8e66f02e",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 29051555,
+              "id": "Q29051555"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "Q34987$1D3FF6BD-4B52-4687-8499-619F3B973DEC",
+        "rank": "normal",
+        "references": [
+          {
+            "hash": "816f6cabdb11971f95dc3faca697f5c2f6f02248",
+            "snaks": {
+              "P248": [
+                {
+                  "snaktype": "value",
+                  "property": "P248",
+                  "hash": "375610daaf9b20dda81cd83553d79636108b64cb",
+                  "datavalue": {
+                    "value": {
+                      "entity-type": "item",
+                      "numeric-id": 14790,
+                      "id": "Q14790"
+                    },
+                    "type": "wikibase-entityid"
+                  },
+                  "datatype": "wikibase-item"
+                }
+              ],
+              "P813": [
+                {
+                  "snaktype": "value",
+                  "property": "P813",
+                  "hash": "02f847ace2885e56ef89186fe9e65776dfd4eb44",
+                  "datavalue": {
+                    "value": {
+                      "time": "+2019-05-10T00:00:00Z",
+                      "timezone": 0,
+                      "before": 0,
+                      "after": 0,
+                      "precision": 11,
+                      "calendarmodel": "http://www.wikidata.org/entity/Q1985727"
+                    },
+                    "type": "time"
+                  },
+                  "datatype": "time"
+                }
+              ],
+              "P854": [
+                {
+                  "snaktype": "value",
+                  "property": "P854",
+                  "hash": "304d2db566e4f943c5b027dce032d0bff1ef0efd",
+                  "datavalue": {
+                    "value": "https://www.ethnologue.com/language/bil",
+                    "type": "string"
+                  },
+                  "datatype": "url"
+                }
+              ]
+            },
+            "snaks-order": [
+              "P248",
+              "P813",
+              "P854"
+            ]
+          }
+        ]
+      }
+    ],
+    "P17": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P17",
+          "hash": "8116beb62ea32777b0b68313efb65d9b2713adc7",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 1033,
+              "id": "Q1033"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "Q34987$D5976F14-7A23-4505-BEEA-EC4664CBAD2F",
+        "rank": "normal"
+      }
+    ],
+    "P2341": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P2341",
+          "hash": "7d0912c4f9e250997b8c2bc48b5716aa5d07a309",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 337514,
+              "id": "Q337514"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "Q34987$49FFB340-8004-4E83-80F1-633480A009B2",
+        "rank": "normal"
+      },
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P2341",
+          "hash": "1c831bf6c99e4eee1187c79f94909cab30cefa1c",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 509300,
+              "id": "Q509300"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "Q34987$AAA30F5B-FD53-48CF-9579-8C24E1F7C573",
+        "rank": "normal"
+      }
+    ],
+    "P2888": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P2888",
+          "hash": "b73d33b1e40b47d117a1a4bf521d8f4f91184474",
+          "datavalue": {
+            "value": "http://publications.europa.eu/resource/authority/language/BIL",
+            "type": "string"
+          },
+          "datatype": "url"
+        },
+        "type": "statement",
+        "id": "Q34987$46734317-5E4C-4CE3-A43E-514BA7D702ED",
+        "rank": "normal",
+        "references": [
+          {
+            "hash": "56b370ad342ff9d47a4119b3f53f894995cea4b7",
+            "snaks": {
+              "P248": [
+                {
+                  "snaktype": "value",
+                  "property": "P248",
+                  "hash": "27fbd18e13a0a2c1e2ae172bfbfc66577ffb38a2",
+                  "datavalue": {
+                    "value": {
+                      "entity-type": "item",
+                      "numeric-id": 75488338,
+                      "id": "Q75488338"
+                    },
+                    "type": "wikibase-entityid"
+                  },
+                  "datatype": "wikibase-item"
+                }
+              ],
+              "P854": [
+                {
+                  "snaktype": "value",
+                  "property": "P854",
+                  "hash": "68a06f5f5ab362597f2d12e45856619795420364",
+                  "datavalue": {
+                    "value": "https://op.europa.eu/web/eu-vocabularies/at-dataset/-/resource/dataset/language",
+                    "type": "string"
+                  },
+                  "datatype": "url"
+                }
+              ]
+            },
+            "snaks-order": [
+              "P248",
+              "P854"
+            ]
+          }
+        ]
+      }
+    ]
+  },
+  "sitelinks": {
+    "dewiki": {
+      "site": "dewiki",
+      "title": "Bile (Sprache)",
+      "badges": []
+    },
+    "enwiki": {
+      "site": "enwiki",
+      "title": "Bile language",
+      "badges": []
+    },
+    "hrwiki": {
+      "site": "hrwiki",
+      "title": "Bile jezik",
+      "badges": []
+    },
+    "pmswiki": {
+      "site": "pmswiki",
+      "title": "Lenga Bile",
+      "badges": []
+    },
+    "ruwiki": {
+      "site": "ruwiki",
+      "title": "Биле",
+      "badges": []
+    },
+    "swwiki": {
+      "site": "swwiki",
+      "title": "Kibile",
+      "badges": []
+    }
+  }
+}
diff --git a/wdtk-datamodel/src/test/resources/JsonDeserializer/lexeme.json b/wdtk-datamodel/src/test/resources/JsonDeserializer/lexeme.json
new file mode 100644
index 00000000..cc7651d9
--- /dev/null
+++ b/wdtk-datamodel/src/test/resources/JsonDeserializer/lexeme.json
@@ -0,0 +1,79 @@
+{
+  "pageid": 55199558,
+  "ns": 146,
+  "title": "Lexeme:L3872",
+  "lastrevid": 1080404095,
+  "modified": "2019-12-20T14:47:45Z",
+  "type": "lexeme",
+  "id": "L3872",
+  "lemmas": {
+    "en": {
+      "language": "en",
+      "value": "business"
+    }
+  },
+  "lexicalCategory": "Q1084",
+  "language": "Q1860",
+  "claims": {},
+  "forms": [
+    {
+      "id": "L3872-F1",
+      "representations": {
+        "en": {
+          "language": "en",
+          "value": "business"
+        }
+      },
+      "grammaticalFeatures": [
+        "Q110786"
+      ],
+      "claims": []
+    },
+    {
+      "id": "L3872-F2",
+      "representations": {
+        "en": {
+          "language": "en",
+          "value": "businesses"
+        }
+      },
+      "grammaticalFeatures": [
+        "Q146786"
+      ],
+      "claims": []
+    }
+  ],
+  "senses": [
+    {
+      "id": "L3872-S1",
+      "glosses": {
+        "en": {
+          "language": "en",
+          "value": "economic activity done by a businessperson"
+        }
+      },
+      "claims": {
+        "P5137": [
+          {
+            "mainsnak": {
+              "snaktype": "value",
+              "property": "P5137",
+              "hash": "e4c2933a6e50281693686282fb92e64b41fd80b1",
+              "datavalue": {
+                "value": {
+                  "entity-type": "item",
+                  "numeric-id": 19862406,
+                  "id": "Q19862406"
+                },
+                "type": "wikibase-entityid"
+              }
+            },
+            "type": "statement",
+            "id": "L3872-S1$14106EF8-9525-41FA-A358-285A314276EA",
+            "rank": "normal"
+          }
+        ]
+      }
+    }
+  ]
+}
diff --git a/wdtk-datamodel/src/test/resources/JsonDeserializer/mediainfo.json b/wdtk-datamodel/src/test/resources/JsonDeserializer/mediainfo.json
new file mode 100644
index 00000000..53748987
--- /dev/null
+++ b/wdtk-datamodel/src/test/resources/JsonDeserializer/mediainfo.json
@@ -0,0 +1,51 @@
+{
+  "pageid": 74698470,
+  "ns": 6,
+  "title": "File:Chick Corea & Stanley Clarke.jpg",
+  "lastrevid": 363818804,
+  "modified": "2019-08-30T13:07:37Z",
+  "type": "mediainfo",
+  "id": "M74698470",
+  "labels": {},
+  "descriptions": {},
+  "statements": {
+    "P180": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P180",
+          "hash": "a83bcd35f5bd70a205d9eabf429841a6a091d973",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 192465,
+              "id": "Q192465"
+            },
+            "type": "wikibase-entityid"
+          }
+        },
+        "type": "statement",
+        "id": "M74698470$c598e90e-44b9-6214-64aa-367e4b2415b6",
+        "rank": "normal"
+      },
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P180",
+          "hash": "988edf83d80d66fd97714b6c977f5f3097ee194d",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 453406,
+              "id": "Q453406"
+            },
+            "type": "wikibase-entityid"
+          }
+        },
+        "type": "statement",
+        "id": "M74698470$209cb38c-4471-7fc6-5a0e-8232ab7a506c",
+        "rank": "normal"
+      }
+    ]
+  }
+}
diff --git a/wdtk-datamodel/src/test/resources/JsonDeserializer/property.json b/wdtk-datamodel/src/test/resources/JsonDeserializer/property.json
new file mode 100644
index 00000000..d8e0e521
--- /dev/null
+++ b/wdtk-datamodel/src/test/resources/JsonDeserializer/property.json
@@ -0,0 +1,291 @@
+{
+  "pageid": 30034564,
+  "ns": 120,
+  "title": "Property:P3467",
+  "lastrevid": 830819467,
+  "modified": "2019-01-09T10:01:52Z",
+  "type": "property",
+  "datatype": "external-id",
+  "id": "P3467",
+  "labels": {
+    "en": {
+      "language": "en",
+      "value": "Inventario Sculture - Polo Museale Fiorentino"
+    },
+    "fr": {
+      "language": "fr",
+      "value": "identifiant Inventario Sculture"
+    },
+    "de": {
+      "language": "de",
+      "value": "Inventario Sculture - Polo Museale Fiorentino"
+    },
+    "nl": {
+      "language": "nl",
+      "value": "Inventario Sculture - Polo Museale Fiorentino-identificatiecode"
+    },
+    "it": {
+      "language": "it",
+      "value": "Inventario Sculture - Polo Museale Fiorentino"
+    },
+    "uk": {
+      "language": "uk",
+      "value": "ідентифікатор Inventario Sculture - Polo Museale Fiorentino"
+    },
+    "ar": {
+      "language": "ar",
+      "value": "مخزون تماثيل متاحف فلورنسا"
+    }
+  },
+  "descriptions": {
+    "en": {
+      "language": "en",
+      "value": "identifier of an artwork in the inventory of sculptures of Florentine museums"
+    },
+    "fr": {
+      "language": "fr",
+      "value": "identifiant d'une œuvre d'art dans l'inventaire des musée de Florence"
+    },
+    "de": {
+      "language": "de",
+      "value": "Identifikator im Skulpturenverzeichnis der Florentiner Museen"
+    },
+    "uk": {
+      "language": "uk",
+      "value": "ідентифікатор твору в реєстрі скульптур флорентійських музеїв"
+    }
+  },
+  "aliases": {
+    "fr": [
+      {
+        "language": "fr",
+        "value": "identifiant Musées florentins dans l'inventaire des sculptures"
+      }
+    ],
+    "ar": [
+      {
+        "language": "ar",
+        "value": "تماثيل متاحف فلورنسا"
+      },
+      {
+        "language": "ar",
+        "value": "متاحف فلورنسا"
+      }
+    ]
+  },
+  "claims": {
+    "P1630": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P1630",
+          "hash": "9b277739598a6d628ba53d6f60715f9710a02da0",
+          "datavalue": {
+            "value": "http://www.polomuseale.firenze.it/invSculture/scheda.asp?position=1&ninv=$1",
+            "type": "string"
+          },
+          "datatype": "string"
+        },
+        "type": "statement",
+        "id": "P3467$ae93c71e-4136-76e7-fe65-8a9b7b56fc91",
+        "rank": "normal"
+      }
+    ],
+    "P1855": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P1855",
+          "hash": "97a73a7c139f349a76231c5a859b8135751d9da0",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 179900,
+              "id": "Q179900"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "qualifiers": {
+          "P3467": [
+            {
+              "snaktype": "value",
+              "property": "P3467",
+              "hash": "78e7e897edc03f2a240745407bb7a72c908b90b2",
+              "datavalue": {
+                "value": "1076",
+                "type": "string"
+              },
+              "datatype": "external-id"
+            }
+          ]
+        },
+        "qualifiers-order": [
+          "P3467"
+        ],
+        "id": "P3467$47877943-4a51-e3cb-ebf3-994a93462c16",
+        "rank": "normal"
+      }
+    ],
+    "P1659": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P1659",
+          "hash": "995c8df1850c9b9ba0b64e046e87589f6b2166b8",
+          "datavalue": {
+            "value": {
+              "entity-type": "property",
+              "numeric-id": 1726,
+              "id": "P1726"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-property"
+        },
+        "type": "statement",
+        "id": "P3467$3207cb44-4f32-1d3f-a528-26263d9c23b2",
+        "rank": "normal"
+      },
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P1659",
+          "hash": "5350e2f34085bd8efe0fc409f6fafc8e2a19ab66",
+          "datavalue": {
+            "value": {
+              "entity-type": "property",
+              "numeric-id": 3504,
+              "id": "P3504"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-property"
+        },
+        "type": "statement",
+        "id": "P3467$f4e38de1-4e20-d61a-df96-2712a2d06968",
+        "rank": "normal"
+      }
+    ],
+    "P31": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P31",
+          "hash": "f4decee31e9752960d9623ea58d1dfd672b31341",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 19847637,
+              "id": "Q19847637"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "P3467$20910560-30AC-45BA-A59A-3D360FFCD8F1",
+        "rank": "normal"
+      },
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P31",
+          "hash": "1f57f44f72b149c4f7fca8eedaf8f07f7ca3324f",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 44847669,
+              "id": "Q44847669"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "P3467$861b3b1c-4456-a89d-609a-920d583accde",
+        "rank": "normal"
+      },
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P31",
+          "hash": "c65038907f055c45440c54a515d4b0d2962edf25",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 45312863,
+              "id": "Q45312863"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "P3467$e3c68b14-41b3-68fb-8311-6360b1f2a90b",
+        "rank": "normal"
+      }
+    ],
+    "P3254": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P3254",
+          "hash": "edbfa6b14cc828f6026a70112475cb16704a5a1a",
+          "datavalue": {
+            "value": "https://www.wikidata.org/wiki/Wikidata:Property_proposal/Inventario_Sculture_-_Polo_Museale_Fiorentino",
+            "type": "string"
+          },
+          "datatype": "url"
+        },
+        "type": "statement",
+        "id": "P3467$FE8D4091-AABB-4445-98C6-DF894971F5C4",
+        "rank": "normal"
+      }
+    ],
+    "P17": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P17",
+          "hash": "b48529e9f7e0898ab1ddaefe8547cdb863e0167c",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 38,
+              "id": "Q38"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "P3467$e4c3c067-48ba-cdae-aae7-3cf6295abf67",
+        "rank": "normal"
+      }
+    ],
+    "P2875": [
+      {
+        "mainsnak": {
+          "snaktype": "value",
+          "property": "P2875",
+          "hash": "dbf395280e2e27dbd093e482bb2915baae993e77",
+          "datavalue": {
+            "value": {
+              "entity-type": "item",
+              "numeric-id": 45312584,
+              "id": "Q45312584"
+            },
+            "type": "wikibase-entityid"
+          },
+          "datatype": "wikibase-item"
+        },
+        "type": "statement",
+        "id": "P3467$91df8651-4147-8940-be42-22d7e30dd1b4",
+        "rank": "normal"
+      }
+    ]
+  }
+}