Repository: Wikidata/Wikidata-Toolkit (java)

Issue:
JSON dump parsing error while running examples
I encountered below error message when running the examples `GreatestNumberProcessor` and `GenderRatioProcessor`.   
```
Problematic line was: {"type":"lexeme","id":"L34069","lemmas":{"de":{"la...
Error when reading JSON for entity: Missing type id when trying to resolve subtype of [simple type, class org.wikidata.wdtk.datamodel.implementation.FormDocumentImpl]: missing type id property 'type' (for POJO property 'forms')
 at [Source: (String)"{"type":"lexeme","id":"L34070","lemmas":{"de":{"language":"de","value":"Zahnl\u00fccke"}},"lexicalCategory":"Q1084","language":"Q188","claims":{"P5185":[{"mainsnak":{"snaktype":"value","property":"P5185","datavalue":{"value":{"entity-type":"item","numeric-id":1775415,"id":"Q1775415"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"L34070$B9D7523B-8AE6-4C39-85A2-EE7CC71C5228","rank":"normal"}],"P8376":[{"mainsnak":{"snaktype":"value","property":"P8376","datavalue""[truncated 4073 chars]; line: 1, column: 1043] (through reference chain: org.wikidata.wdtk.datamodel.implementation.LexemeDocumentImpl["forms"]->java.util.ArrayList[0])
```
The dump is automatically downloaded `20210407.json.gz`.  
  
I tried to run the code on Windows (Build with Maven with IDEA, JDK 15.0.1) and Ubuntu (openJDK 1.8.0_282, run as IDEA remote). 

It seems the dump file size is only ~200MB. Would that be correct?

Any help is appreciated. Thank you.

Pull request:
Fix deserialization of lexemes to accept bogus JSON produced by Wikibase
Closes #568.

Patch:
diff --git a/pom.xml b/pom.xml
index 3f1a0c00..acc7b28f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -80,7 +80,7 @@
 		<hamcrestVersion>2.2</hamcrestVersion>
 		<mockitoVersion>4.9.0</mockitoVersion>
 		<rdf4jVersion>3.7.7</rdf4jVersion>
-		<slf4jVersion>2.0.5</slf4jVersion>
+		<slf4jVersion>2.0.6</slf4jVersion>
 		<threetenVersion>1.7.1</threetenVersion>
 		<okhttpSignpostVersion>1.1.0</okhttpSignpostVersion>
 		<okhttpVersion>4.10.0</okhttpVersion>
diff --git a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/LexemeDeserializer.java b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/LexemeDeserializer.java
new file mode 100644
index 00000000..a4b4a0ec
--- /dev/null
+++ b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/helpers/LexemeDeserializer.java
@@ -0,0 +1,123 @@
+package org.wikidata.wdtk.datamodel.helpers;
+
+/*-
+ * #%L
+ * Wikidata Toolkit Data Model
+ * %%
+ * Copyright (C) 2014 - 2022 Wikidata Toolkit Developers
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.exc.InvalidFormatException;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.type.TypeFactory;
+import org.wikidata.wdtk.datamodel.implementation.*;
+import org.wikidata.wdtk.datamodel.interfaces.FormDocument;
+import org.wikidata.wdtk.datamodel.interfaces.LexemeDocument;
+import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
+import org.wikidata.wdtk.datamodel.interfaces.SenseDocument;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Custom deserializer, very close to the default serializer generated by Jackson,
+ * only to also accept empty json objects ({}) in place of empty lists for the list of senses.
+ * https://github.com/Wikidata/Wikidata-Toolkit/issues/568
+ */
+public class LexemeDeserializer extends StdDeserializer<LexemeDocumentImpl> {
+
+    public LexemeDeserializer() {
+        super(LexemeDocumentImpl.class);
+    }
+
+    @Override
+    public LexemeDocumentImpl deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException {
+        JsonNode node = jsonParser.getCodec().readTree(jsonParser);
+        if (! (node instanceof ObjectNode)) {
+            throw new IOException("Deserializing a lexeme can only be done from a JSON object");
+        }
+        ObjectNode object = (ObjectNode) node;
+
+        String jsonId = object.get("id").asText();
+        String lexicalCategory = object.get("lexicalCategory").asText();
+        String language = object.get("language").asText();
+        TypeFactory typeFactory = deserializationContext.getTypeFactory();
+
+        JsonNode lemmas1 = object.get("lemmas");
+        Map<String, TermImpl> preLemmas = deserializationContext.readTreeAsValue(
+                lemmas1,
+                typeFactory.constructMapType(Map.class, String.class, TermImpl.class));
+        Map<String, MonolingualTextValue> lemmas = null;
+        if (preLemmas != null) {
+            lemmas = preLemmas.entrySet()
+                    .stream()
+                    .collect(Collectors.toMap(Map.Entry::getKey, v -> v.getValue()));
+        }
+
+        Map<String, List<StatementImpl.PreStatement>> claims = deserializationContext.readTreeAsValue(
+                object.get("claims"),
+                typeFactory.constructMapType(Map.class, typeFactory.constructType(String.class),
+                        typeFactory.constructCollectionType(List.class, StatementImpl.PreStatement.class)));
+
+        List<FormDocument> forms;
+        if (object.get("forms") instanceof ObjectNode && ((ObjectNode)object.get("forms")).isEmpty()) {
+            forms = Collections.emptyList();
+        } else {
+            forms = deserializationContext.readTreeAsValue(
+                    object.get("forms"),
+                    typeFactory.constructCollectionType(List.class, FormDocumentImpl.class));
+        }
+
+        List<SenseDocument> senses;
+        if (object.get("senses") instanceof ObjectNode && ((ObjectNode)object.get("senses")).isEmpty()) {
+            // case for https://github.com/Wikidata/Wikidata-Toolkit/issues/568
+            senses = Collections.emptyList();
+        } else {
+            senses = deserializationContext.readTreeAsValue(
+                    object.get("senses"),
+                    typeFactory.constructCollectionType(List.class, SenseDocumentImpl.class));
+        }
+
+        long lastrevid = 0;
+        if (object.has("lastrevid")) {
+            lastrevid = object.get("lastrevid").asLong();
+        }
+        String siteIri = (String) deserializationContext.findInjectableValue("siteIri", null , null);
+        return new LexemeDocumentImpl(
+                jsonId,
+                lexicalCategory,
+                language,
+                lemmas,
+                claims,
+                forms,
+                senses,
+                lastrevid,
+                siteIri
+        );
+    }
+}
diff --git a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/LexemeDocumentImpl.java b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/LexemeDocumentImpl.java
index 27f9fc92..96c99f74 100644
--- a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/LexemeDocumentImpl.java
+++ b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/implementation/LexemeDocumentImpl.java
@@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import org.apache.commons.lang3.Validate;
 import org.wikidata.wdtk.datamodel.helpers.Equality;
 import org.wikidata.wdtk.datamodel.helpers.Hash;
+import org.wikidata.wdtk.datamodel.helpers.LexemeDeserializer;
 import org.wikidata.wdtk.datamodel.helpers.ToString;
 import org.wikidata.wdtk.datamodel.interfaces.*;
 
@@ -38,6 +39,7 @@ import java.util.regex.Pattern;
  * @author Thomas Pellissier Tanon
  */
 @JsonIgnoreProperties(ignoreUnknown = true)
+@JsonDeserialize(using = LexemeDeserializer.class)
 public class LexemeDocumentImpl extends StatementDocumentImpl implements LexemeDocument {
 
 	private final ItemIdValue lexicalCategory;
@@ -104,14 +106,14 @@ public class LexemeDocumentImpl extends StatementDocumentImpl implements LexemeD
 	 * deserialization. Should only be used by Jackson for this very purpose.
 	 */
 	@JsonCreator
-	LexemeDocumentImpl(
+	public LexemeDocumentImpl(
 			@JsonProperty("id") String jsonId,
 			@JsonProperty("lexicalCategory") String lexicalCategory,
 			@JsonProperty("language") String language,
-			@JsonProperty("lemmas") @JsonDeserialize(contentAs=TermImpl.class) Map<String, MonolingualTextValue> lemmas,
+			@JsonProperty("lemmas") @JsonDeserialize(contentAs = TermImpl.class) Map<String, MonolingualTextValue> lemmas,
 			@JsonProperty("claims") Map<String, List<StatementImpl.PreStatement>> claims,
-			@JsonProperty("forms") @JsonDeserialize(contentAs=FormDocumentImpl.class) List<FormDocument> forms,
-			@JsonProperty("senses") @JsonDeserialize(contentAs=SenseDocumentImpl.class) List<SenseDocument> senses,
+			@JsonProperty("forms") @JsonDeserialize(contentAs = FormDocumentImpl.class) List<FormDocument> forms,
+			@JsonProperty("senses") @JsonDeserialize(contentAs = SenseDocumentImpl.class) List<SenseDocument> senses,
 			@JsonProperty("lastrevid") long revisionId,
 			@JacksonInject("siteIri") String siteIri) {
 		super(jsonId, claims, revisionId, siteIri);


Test patch:
diff --git a/wdtk-datamodel/src/test/java/org/wikidata/wdtk/datamodel/implementation/LexemeDocumentImplTest.java b/wdtk-datamodel/src/test/java/org/wikidata/wdtk/datamodel/implementation/LexemeDocumentImplTest.java
index a884f39f..2fbc6956 100644
--- a/wdtk-datamodel/src/test/java/org/wikidata/wdtk/datamodel/implementation/LexemeDocumentImplTest.java
+++ b/wdtk-datamodel/src/test/java/org/wikidata/wdtk/datamodel/implementation/LexemeDocumentImplTest.java
@@ -69,8 +69,10 @@ public class LexemeDocumentImplTest {
 
 	private final LexemeDocument ld1 = new LexemeDocumentImpl(lid, lexCat, language, lemmaList, statementGroups, forms, senses, 1234);
 	private final LexemeDocument ld2 = new LexemeDocumentImpl(lid, lexCat, language, lemmaList, statementGroups, forms, senses, 1234);
+	private final LexemeDocument ld3 = new LexemeDocumentImpl(lid, lexCat, language, lemmaList, statementGroups, Collections.emptyList(), Collections.emptyList(), 1234);
 
 	private final String JSON_LEXEME = "{\"type\":\"lexeme\",\"id\":\"L42\",\"lexicalCategory\":\"Q1\",\"language\":\"Q2\",\"lemmas\":{\"en\":{\"language\":\"en\",\"value\":\"lemma\"}},\"claims\":{\"P42\":[{\"rank\":\"normal\",\"id\":\"MyId\",\"mainsnak\":{\"property\":\"P42\",\"snaktype\":\"somevalue\"},\"type\":\"statement\"}]},\"forms\":[{\"type\":\"form\",\"id\":\"L42-F1\",\"representations\":{\"en\":{\"language\":\"en\",\"value\":\"foo\"}},\"grammaticalFeatures\":[],\"claims\":{}}],\"senses\":[{\"type\":\"sense\",\"id\":\"L42-S1\",\"glosses\":{\"en\":{\"language\":\"en\",\"value\":\"foo meaning\"}},\"claims\":{}}],\"lastrevid\":1234}";
+	private final String JSON_LEXEME_FOR_ISSUE_568 = "{\"type\":\"lexeme\",\"id\":\"L42\",\"lexicalCategory\":\"Q1\",\"language\":\"Q2\",\"lemmas\":{\"en\":{\"language\":\"en\",\"value\":\"lemma\"}},\"claims\":{\"P42\":[{\"rank\":\"normal\",\"id\":\"MyId\",\"mainsnak\":{\"property\":\"P42\",\"snaktype\":\"somevalue\"},\"type\":\"statement\"}]},\"forms\":{},\"senses\":{},\"lastrevid\":1234}";
 
 	@Test
 	public void fieldsAreCorrect() {
@@ -306,4 +308,11 @@ public class LexemeDocumentImplTest {
 	public void testLexemeToJava() throws IOException {
 		assertEquals(ld1, mapper.readValue(JSON_LEXEME, LexemeDocumentImpl.class));
 	}
+
+	@Test
+	public void testDeserializeLexemeWithJsonObjectInPlaceOfEmptyList() throws JsonProcessingException {
+		// test for https://github.com/Wikidata/Wikidata-Toolkit/issues/568
+		// phab: https://phabricator.wikimedia.org/T305660
+		assertEquals(ld3, mapper.readValue(JSON_LEXEME_FOR_ISSUE_568, LexemeDocumentImpl.class));
+	}
 }