Repository: wikidata/wikidata-toolkit
Issue: Deserialization failure on dumps with redirects

The Wikidata daily XML dumps contains serialization for redirects.

Wikidata Toolkit 0.7 fails when deserializing them:

``` java
GRAVE: Failed to map JSON for item Q26790101: Unexpected token (END_OBJECT), expected FIELD_NAME: missing property 'type' that is to contain type id  (for class org.wikidata.wdtk.datamodel.json.jackson.JacksonItemDocument)
 at [Source: {"entity":"Q26790101","redirect":"Q21863925"}; line: 1, column: 45]
com.fasterxml.jackson.databind.JsonMappingException: Unexpected token (END_OBJECT), expected FIELD_NAME: missing property 'type' that is to contain type id  (for class org.wikidata.wdtk.datamodel.json.jackson.JacksonItemDocument)
 at [Source: {"entity":"Q26790101","redirect":"Q21863925"}; line: 1, column: 45]
    at com.fasterxml.jackson.databind.JsonMappingException.from(JsonMappingException.java:148)
    at com.fasterxml.jackson.databind.DeserializationContext.wrongTokenException(DeserializationContext.java:927)
    at com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedUsingDefaultImpl(AsPropertyTypeDeserializer.java:151)
    at com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject(AsPropertyTypeDeserializer.java:103)
    at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeWithType(BeanDeserializerBase.java:966)
    at com.fasterxml.jackson.databind.deser.impl.TypeWrappedDeserializer.deserialize(TypeWrappedDeserializer.java:42)
    at com.fasterxml.jackson.databind.ObjectReader._bindAndClose(ObjectReader.java:1443)
    at com.fasterxml.jackson.databind.ObjectReader.readValue(ObjectReader.java:1081)
    at org.wikidata.wdtk.dumpfiles.WikibaseRevisionProcessor.readValue(WikibaseRevisionProcessor.java:160)
    at org.wikidata.wdtk.dumpfiles.WikibaseRevisionProcessor.processItemRevision(WikibaseRevisionProcessor.java:97)
    at org.wikidata.wdtk.dumpfiles.WikibaseRevisionProcessor.processRevision(WikibaseRevisionProcessor.java:88)
    at org.wikidata.wdtk.dumpfiles.MwRevisionProcessorBroker.notifyMwRevisionProcessors(MwRevisionProcessorBroker.java:183)
    at org.wikidata.wdtk.dumpfiles.MwRevisionProcessorBroker.processRevision(MwRevisionProcessorBroker.java:147)
    at org.wikidata.wdtk.dumpfiles.MwRevisionDumpFileProcessor.processXmlRevision(MwRevisionDumpFileProcessor.java:427)
    at org.wikidata.wdtk.dumpfiles.MwRevisionDumpFileProcessor.processXmlPage(MwRevisionDumpFileProcessor.java:345)
    at org.wikidata.wdtk.dumpfiles.MwRevisionDumpFileProcessor.tryProcessXmlPage(MwRevisionDumpFileProcessor.java:282)
    at org.wikidata.wdtk.dumpfiles.MwRevisionDumpFileProcessor.processXmlMediawiki(MwRevisionDumpFileProcessor.java:202)
    at org.wikidata.wdtk.dumpfiles.MwRevisionDumpFileProcessor.processDumpFileContents(MwRevisionDumpFileProcessor.java:155)
    at org.wikidata.wdtk.dumpfiles.DumpProcessingController.processDumpFile(DumpProcessingController.java:546)
    at org.wikidata.wdtk.dumpfiles.DumpProcessingController.processAllRecentRevisionDumps(DumpProcessingController.java:389)
```


Changed files:
wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/WikibaseRevisionProcessor.java

diff --git a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/WikibaseRevisionProcessor.java b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/WikibaseRevisionProcessor.java
index e8d77387..a9c79d3d 100644
--- a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/WikibaseRevisionProcessor.java
+++ b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/WikibaseRevisionProcessor.java
@@ -22,6 +22,7 @@ package org.wikidata.wdtk.dumpfiles;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import org.slf4j.Logger;
@@ -93,6 +94,10 @@ public class WikibaseRevisionProcessor implements MwRevisionProcessor {
 	}
 
 	public void processItemRevision(MwRevision mwRevision) {
+		if(isWikibaseRedirection(mwRevision)) {
+			return;
+		}
+
 		try {
 			JacksonItemDocument document = readValue(mwRevision.getText(), JacksonItemDocument.class);
 			document.setSiteIri(this.siteIri);
@@ -123,6 +128,10 @@ public class WikibaseRevisionProcessor implements MwRevisionProcessor {
 	}
 
 	public void processPropertyRevision(MwRevision mwRevision) {
+		if(isWikibaseRedirection(mwRevision)) {
+			return;
+		}
+
 		try {
 			JacksonPropertyDocument document = readValue(mwRevision.getText(), JacksonPropertyDocument.class);
 			document.setSiteIri(this.siteIri);
@@ -154,6 +163,10 @@ public class WikibaseRevisionProcessor implements MwRevisionProcessor {
 
 	}
 
+	private boolean isWikibaseRedirection(MwRevision mwRevision) {
+		return mwRevision.getText().contains("\\"redirect\\":"); //Hacky but fast
+	}
+
 	public <T> T readValue(String content, Class<T> valueType) throws IOException {
 		return mapper.reader(valueType)
 				.with(DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT)