Repository: wikidata/wikidata-toolkit
Issue: EntityDocumentProcessors called twice when using JSON dumps

Entity processors can be registered for specific content models. However, since JSON dumps do not provide content models, they just call all registered entity document processors on each document. The broker component that manages this does not check for duplicates, and as a result, a processor that is registered individually for item and property content will be called twice.

This is a critical bug that causes unnecessary work and leads to inflated outputs, in particular in the RDF dumps.

As a workaround, users should register their processors for one content model only (or simply for null) when using the JSON dumps.


Changed files:
wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/interfaces/EntityDocumentProcessorBroker.java
wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/DumpProcessingController.java

diff --git a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/interfaces/EntityDocumentProcessorBroker.java b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/interfaces/EntityDocumentProcessorBroker.java
index 9afdff19..3522b65f 100644
--- a/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/interfaces/EntityDocumentProcessorBroker.java
+++ b/wdtk-datamodel/src/main/java/org/wikidata/wdtk/datamodel/interfaces/EntityDocumentProcessorBroker.java
@@ -9,9 +9,9 @@ package org.wikidata.wdtk.datamodel.interfaces;
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,6 +21,7 @@ package org.wikidata.wdtk.datamodel.interfaces;
  */
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 
 /**
@@ -33,17 +34,23 @@ import java.util.List;
 public class EntityDocumentProcessorBroker implements EntityDocumentProcessor {
 
 	final List<EntityDocumentProcessor> entityDocumentProcessors = new ArrayList<EntityDocumentProcessor>();
+	final HashSet<EntityDocumentProcessor> entityDocumentProcessorRegistry = new HashSet<>();
 
 	/**
 	 * Registers a listener which will be called for all entity documents that
-	 * are processed.
+	 * are processed. The method avoids duplicates in the sense that the exact
+	 * same object cannot be registered twice.
 	 *
 	 * @param entityDocumentProcessor
 	 *            the listener to register
 	 */
 	public void registerEntityDocumentProcessor(
 			EntityDocumentProcessor entityDocumentProcessor) {
-		this.entityDocumentProcessors.add(entityDocumentProcessor);
+		if (!this.entityDocumentProcessorRegistry
+				.contains(entityDocumentProcessor)) {
+			this.entityDocumentProcessors.add(entityDocumentProcessor);
+			this.entityDocumentProcessorRegistry.add(entityDocumentProcessor);
+		}
 	}
 
 	@Override
diff --git a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/DumpProcessingController.java b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/DumpProcessingController.java
index 19072f0e..0a4545a7 100644
--- a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/DumpProcessingController.java
+++ b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/DumpProcessingController.java
@@ -9,9 +9,9 @@ package org.wikidata.wdtk.dumpfiles;
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -153,7 +153,7 @@ public class DumpProcessingController {
 	/**
 	 * Broker object to distribute entity documents to several listeners. This
 	 * will be the main object that distributes revisions on any document-based
-	 * processing run.
+	 * processing run (in particular for JSON dumps).
 	 */
 	EntityDocumentProcessorBroker entityDocumentProcessorBroker = new EntityDocumentProcessorBroker();
 
@@ -238,8 +238,11 @@ public class DumpProcessingController {
 	 * Registers an MwRevisionProcessor, which will henceforth be notified of
 	 * all revisions that are encountered in the dump.
 	 * <p>
+	 * This only is used when processing dumps that contain revisions. In
+	 * particular, plain JSON dumps contain no revision information.
+	 * <p>
 	 * Importantly, the {@link MwRevision} that the registered processors will
-	 * receive is is valid only during the execution of
+	 * receive is valid only during the execution of
 	 * {@link MwRevisionProcessor#processRevision(MwRevision)}, but it will not
 	 * be permanent. If the data is to be retained permanently, the revision
 	 * processor needs to make its own copy.
@@ -266,6 +269,12 @@ public class DumpProcessingController {
 	/**
 	 * Registers an EntityDocumentProcessor, which will henceforth be notified
 	 * of all entity documents that are encountered in the dump.
+	 * <p>
+	 * It is possible to register processors for specific content types and to
+	 * use either all revisions or only the most current ones. This
+	 * functionality is only available when processing dumps that contain this
+	 * information. In particular, plain JSON dumps do not specify content
+	 * models at all and have only one (current) revision of each entity.
 	 *
 	 * @param entityDocumentProcessor
 	 *            the entity document processor to register