Repository: Wikidata/Wikidata-Toolkit (java)

Issue:
WmfOnlineDailyDumpFile incorrectly checks for availability
When trying to use this toolkit to manage downloading dumps, I encountered an issue on the way they are determined to be available.
The code responsible for this is :

```java
protected boolean fetchIsDone() {
		boolean result;
		try (InputStream in = this.webResourceFetcher
				.getInputStreamForUrl(getBaseUrl() + "status.txt")) {
			BufferedReader bufferedReader = new BufferedReader(
					new InputStreamReader(in, StandardCharsets.UTF_8));
			String inputLine = bufferedReader.readLine();
			bufferedReader.close();
			result = "done".equals(inputLine);
		} catch (IOException e) { // file not found or not readable
			result = false;
		}
		return result;
	}
```

However, when checking what is provided by the WMF, we see that status.txt doesn't show just done anymore but done:all (and perhaps other, I haven't made an exhaustive check), see: https://dumps.wikimedia.org/other/incr/wikidatawiki/20240414/status.txt

Would it possible to update the `"done".equals(inputLine);` so it is correct ? (perhaps with `startsWith ?`)


Pull request:
Fix online dump downloads for the various modes and Closes #869
It seems that there were changes in the way dumps are structured for wikidata making the way dump files were automatically downloaded not functional anymore.

Here are the fixes implemented, and the reasoning behind it :
DAILY : update the check for an available dump as status.txt now indicated "done:all". See : https://dumps.wikimedia.org/other/incr/wikidatawiki/20240419/status.txt I put startsWith but it might be preferable to check for done:all equality ? I don't know the various possible values but this version should work at least most of the time (compared to none right now).
JSON : The JSON dumps are currently downloaded from this folder : https://dumps.wikimedia.org/other/wikidata/ It is a bit of a mess including both full and incremental (?) dumps. I switch it to use https://dumps.wikimedia.org/wikidatawiki/entities/ (which seems to be equivalent to https://dumps.wikimedia.org/other/wikibase/wikidatawiki/, I don't know if anyone has any insight into which link should be preferred). Moreover, not all shown dates feature the full json dump so I added a check that the file is actually there (I have not found a status.txt or equivalent for the entitity dump)
SITES : Nothing done, haven't tested it but the file it is looking for is still there so it probably still works.
CURRENT / FULL : Both of these look for a -pages-meta-current.xml.bz2 / -pages-meta-history.xml.bz2 inside a dump (like https://dumps.wikimedia.org/wikidatawiki/20240401/). However these files do not seem to be there. Nevertheless, parts of these file ARE present (of form https://dumps.wikimedia.org/wikidatawiki/20240401/wikidatawiki-20240401-pages-meta-current1.xml-p1p441397.bz2 for example) but not the united file. If anyone knows why the full file isn't there I could try to fix the issue


Patch:
diff --git a/pom.xml b/pom.xml
index 6f187c2e..f786b64e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,7 +72,7 @@
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<jacksonVersion>2.17.0</jacksonVersion>
+		<jacksonVersion>2.17.1</jacksonVersion>
 		<okhttpVersion>4.12.0</okhttpVersion>
 	</properties>
 
@@ -93,7 +93,7 @@
 			<dependency>
 				<groupId>org.mockito</groupId>
 				<artifactId>mockito-core</artifactId>
-				<version>5.11.0</version>
+				<version>5.12.0</version>
 				<scope>test</scope>
 			</dependency>
 			<dependency>
@@ -114,7 +114,7 @@
 			<dependency>
 				<groupId>org.apache.commons</groupId>
 				<artifactId>commons-compress</artifactId>
-				<version>1.26.1</version>
+				<version>1.26.2</version>
 			</dependency>
 			<dependency>
 				<groupId>com.fasterxml.jackson.core</groupId>
@@ -171,12 +171,12 @@
 			<dependency>
 				<groupId>org.eclipse.rdf4j</groupId>
 				<artifactId>rdf4j-rio-turtle</artifactId>
-				<version>4.3.11</version>
+				<version>4.3.12</version>
 			</dependency>
 			<dependency>
 				<groupId>org.eclipse.rdf4j</groupId>
 				<artifactId>rdf4j-rio-ntriples</artifactId>
-				<version>4.3.11</version>
+				<version>4.3.12</version>
 			</dependency>
 		</dependencies>
 	</dependencyManagement>
@@ -325,7 +325,7 @@
 				<!-- Plugin for creating Javadocs; goal for preparing docs for upload to github: javadoc:aggregate -->
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-javadoc-plugin</artifactId>
-				<version>3.6.3</version>
+				<version>3.7.0</version>
 				<configuration>
 					<bottom>
 						<![CDATA[Copyright &#169; {inceptionYear}&#x2013;{currentYear} {organizationName}. Generated from source code published under the Apache License 2.0. For more information, see the <a href="https://www.mediawiki.org/wiki/Wikidata_Toolkit">Wikidata Toolkit homepage</a>]]></bottom>
@@ -344,7 +344,7 @@
 				<!-- Staging plugin -->
 				<groupId>org.sonatype.plugins</groupId>
 				<artifactId>nexus-staging-maven-plugin</artifactId>
-				<version>1.6.13</version>
+				<version>1.7.0</version>
 				<extensions>true</extensions>
 				<configuration>
 					<serverId>ossrh</serverId>
@@ -356,7 +356,7 @@
 				<!-- Signature -->
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-gpg-plugin</artifactId>
-				<version>3.2.3</version>
+				<version>3.2.4</version>
 				<executions>
 					<execution>
 						<id>sign-artifacts</id>
diff --git a/wdtk-distribution/pom.xml b/wdtk-distribution/pom.xml
index ffd9ac33..a4562d1d 100644
--- a/wdtk-distribution/pom.xml
+++ b/wdtk-distribution/pom.xml
@@ -44,7 +44,7 @@
 			      <plugin>
 			        <groupId>org.apache.maven.plugins</groupId>
 			        <artifactId>maven-shade-plugin</artifactId>
-			        <version>3.5.2</version>
+			        <version>3.6.0</version>
 			        <executions>
 			          <execution>
 			          	<id>wdtk-library</id>
diff --git a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/JsonOnlineDumpFile.java b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/JsonOnlineDumpFile.java
index d6f35149..f76480be 100644
--- a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/JsonOnlineDumpFile.java
+++ b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/JsonOnlineDumpFile.java
@@ -22,12 +22,16 @@ package org.wikidata.wdtk.dumpfiles.wmf;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.wikidata.wdtk.dumpfiles.DumpContentType;
 import org.wikidata.wdtk.util.DirectoryManager;
 import org.wikidata.wdtk.util.WebResourceFetcher;
+import org.wikidata.wdtk.util.WebResourceFetcherImpl;
 
 public class JsonOnlineDumpFile extends WmfDumpFile {
 
@@ -115,9 +119,22 @@ public class JsonOnlineDumpFile extends WmfDumpFile {
 
 	@Override
 	protected boolean fetchIsDone() {
-		// WMF provides no easy way to check this for these files;
-		// so just assume it is done
-		return true;
+		boolean result = false;
+		String fileName = WmfDumpFile.getDumpFileName(DumpContentType.JSON,
+				this.projectName, this.dateStamp);
+		String urlString = getBaseUrl() + fileName;
+		// make a head request to check if resource exists
+		try {
+			HttpURLConnection connection = (HttpURLConnection) WebResourceFetcherImpl.getUrlConnection(new URL(urlString));
+			connection.setRequestMethod("HEAD");
+			int responseCode = connection.getResponseCode();
+			if (responseCode == HttpURLConnection.HTTP_OK) {
+				result = true;
+			}
+		} catch (IOException e) { // file not found or not readable
+			result = false;
+		}
+		return result;
 	}
 
 	/**
@@ -127,7 +144,7 @@ public class JsonOnlineDumpFile extends WmfDumpFile {
 	 */
 	String getBaseUrl() {
 		return WmfDumpFile.getDumpFileWebDirectory(DumpContentType.JSON,
-				this.projectName);
+				this.projectName) + this.dateStamp + "/";
 	}
 
 }
diff --git a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/WmfDumpFile.java b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/WmfDumpFile.java
index 08c12450..604ed302 100644
--- a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/WmfDumpFile.java
+++ b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/WmfDumpFile.java
@@ -53,7 +53,7 @@ public abstract class WmfDumpFile implements MwDumpFile {
 		WmfDumpFile.WEB_DIRECTORY.put(DumpContentType.CURRENT, "");
 		WmfDumpFile.WEB_DIRECTORY.put(DumpContentType.FULL, "");
 		WmfDumpFile.WEB_DIRECTORY.put(DumpContentType.SITES, "");
-		WmfDumpFile.WEB_DIRECTORY.put(DumpContentType.JSON, "other/");
+		WmfDumpFile.WEB_DIRECTORY.put(DumpContentType.JSON, "");
 	}
 
 	/**
@@ -68,7 +68,7 @@ public abstract class WmfDumpFile implements MwDumpFile {
 		WmfDumpFile.POSTFIXES.put(DumpContentType.FULL,
 				"-pages-meta-history.xml.bz2");
 		WmfDumpFile.POSTFIXES.put(DumpContentType.SITES, "-sites.sql.gz");
-		WmfDumpFile.POSTFIXES.put(DumpContentType.JSON, ".json.gz");
+		WmfDumpFile.POSTFIXES.put(DumpContentType.JSON, "-all.json.gz");
 	}
 
 	/**
@@ -172,7 +172,7 @@ public abstract class WmfDumpFile implements MwDumpFile {
 			if ("wikidatawiki".equals(projectName)) {
 				return WmfDumpFile.DUMP_SITE_BASE_URL
 						+ WmfDumpFile.WEB_DIRECTORY.get(dumpContentType)
-						+ "wikidata" + "/";
+						+ projectName + "/entities/";
 			} else {
 				throw new RuntimeException(
 						"Wikimedia Foundation uses non-systematic directory names for this type of dump file."
@@ -253,7 +253,7 @@ public abstract class WmfDumpFile implements MwDumpFile {
 	public static String getDumpFileName(DumpContentType dumpContentType,
 			String projectName, String dateStamp) {
 		if (dumpContentType == DumpContentType.JSON) {
-			return dateStamp + WmfDumpFile.getDumpFilePostfix(dumpContentType);
+			return "wikidata-" + dateStamp + WmfDumpFile.getDumpFilePostfix(dumpContentType);
 		} else {
 			return projectName + "-" + dateStamp
 					+ WmfDumpFile.getDumpFilePostfix(dumpContentType);
diff --git a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/WmfOnlineDailyDumpFile.java b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/WmfOnlineDailyDumpFile.java
index 5065a8ba..8f8eb7d4 100644
--- a/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/WmfOnlineDailyDumpFile.java
+++ b/wdtk-dumpfiles/src/main/java/org/wikidata/wdtk/dumpfiles/wmf/WmfOnlineDailyDumpFile.java
@@ -137,7 +137,7 @@ class WmfOnlineDailyDumpFile extends WmfDumpFile {
 					new InputStreamReader(in, StandardCharsets.UTF_8));
 			String inputLine = bufferedReader.readLine();
 			bufferedReader.close();
-			result = "done".equals(inputLine);
+			result = inputLine.startsWith("done");
 		} catch (IOException e) { // file not found or not readable
 			result = false;
 		}


Test patch:
diff --git a/wdtk-dumpfiles/src/test/java/org/wikidata/wdtk/dumpfiles/JsonDumpFileProcessingTest.java b/wdtk-dumpfiles/src/test/java/org/wikidata/wdtk/dumpfiles/JsonDumpFileProcessingTest.java
index f9721aa7..8d9eabde 100644
--- a/wdtk-dumpfiles/src/test/java/org/wikidata/wdtk/dumpfiles/JsonDumpFileProcessingTest.java
+++ b/wdtk-dumpfiles/src/test/java/org/wikidata/wdtk/dumpfiles/JsonDumpFileProcessingTest.java
@@ -169,7 +169,7 @@ public class JsonDumpFileProcessingTest {
 		Path dumpFilePath = dmPath.resolve("dumpfiles").resolve("wikidatawiki");
 		Path thisDumpPath = dumpFilePath.resolve(dumpContentType.toString()
 				.toLowerCase() + "-" + dateStamp);
-		Path filePath = thisDumpPath.resolve(dateStamp + WmfDumpFile.getDumpFilePostfix(dumpContentType));
+		Path filePath = thisDumpPath.resolve("wikidata-" + dateStamp + WmfDumpFile.getDumpFilePostfix(dumpContentType));
 		dm.setFileContents(filePath, MockStringContentFactory.getStringFromUrl(resourceUrl),
 				WmfDumpFile.getDumpFileCompressionType(filePath.toString()));
 	}
diff --git a/wdtk-dumpfiles/src/test/java/org/wikidata/wdtk/dumpfiles/wmf/WmfDumpFileManagerTest.java b/wdtk-dumpfiles/src/test/java/org/wikidata/wdtk/dumpfiles/wmf/WmfDumpFileManagerTest.java
index b3b41c0c..717f76dd 100644
--- a/wdtk-dumpfiles/src/test/java/org/wikidata/wdtk/dumpfiles/wmf/WmfDumpFileManagerTest.java
+++ b/wdtk-dumpfiles/src/test/java/org/wikidata/wdtk/dumpfiles/wmf/WmfDumpFileManagerTest.java
@@ -38,6 +38,7 @@ import org.wikidata.wdtk.dumpfiles.MwDumpFileProcessor;
 import org.wikidata.wdtk.testing.MockDirectoryManager;
 import org.wikidata.wdtk.testing.MockStringContentFactory;
 import org.wikidata.wdtk.testing.MockWebResourceFetcher;
+import org.wikidata.wdtk.util.WebResourceFetcherImpl;
 
 public class WmfDumpFileManagerTest {
 
@@ -61,7 +62,8 @@ public class WmfDumpFileManagerTest {
 			try {
 				result = result
 						+ MockStringContentFactory
-								.getStringFromInputStream(inputStream) + "\n";
+								.getStringFromInputStream(inputStream)
+						+ "\n";
 			} catch (IOException e) {
 				throw new RuntimeException(e);
 			}
@@ -146,9 +148,7 @@ public class WmfDumpFileManagerTest {
 
 	@Test
 	public void getAllJsonDumps() throws IOException {
-		wrf.setWebResourceContentsFromResource(
-				"https://dumps.wikimedia.org/other/wikidata/",
-				"/other-wikidata-index.html", this.getClass());
+		WebResourceFetcherImpl onlineWrf = new WebResourceFetcherImpl();
 
 		setLocalDump("20141110", DumpContentType.JSON, true);
 		setLocalDump("20150105", DumpContentType.CURRENT, true);
@@ -156,23 +156,24 @@ public class WmfDumpFileManagerTest {
 		setLocalDump("nodate", DumpContentType.JSON, true);
 
 		WmfDumpFileManager dumpFileManager = new WmfDumpFileManager(
-				"wikidatawiki", dm, wrf);
+				"wikidatawiki", dm, onlineWrf);
 
 		List<? extends MwDumpFile> dumpFiles = dumpFileManager
 				.findAllDumps(DumpContentType.JSON);
 
-		String[] dumpDates = { "20150112", "20150105", "20141229", "20141222",
-				"20141215", "20141210", "20141201", "20141124", "20141117",
-				"20141110" };
-		boolean[] dumpIsLocal = { false, false, false, false, false, false,
-				true, false, false, true };
+		String[] localDumpDates = { "20141201", "20141110" };
 
-		assertEquals(dumpFiles.size(), dumpDates.length);
+		assertTrue(localDumpDates.length < dumpFiles.size());
 		for (int i = 0; i < dumpFiles.size(); i++) {
 			assertEquals(dumpFiles.get(i).getDumpContentType(),
 					DumpContentType.JSON);
-			assertEquals(dumpFiles.get(i).getDateStamp(), dumpDates[i]);
-			if (dumpIsLocal[i]) {
+			boolean shouldBeLocal = false;
+			for (String localDumpDate : localDumpDates) {
+				if (localDumpDate.equals(dumpFiles.get(i).getDateStamp())) {
+					shouldBeLocal = true;
+				}
+			}
+			if (shouldBeLocal) {
 				assertTrue(
 						"Dumpfile " + dumpFiles.get(i) + " should be local.",
 						dumpFiles.get(i) instanceof WmfLocalDumpFile);
@@ -184,6 +185,21 @@ public class WmfDumpFileManagerTest {
 		}
 	}
 
+	@Test
+	public void findMostRecentJsonDump() throws IOException {
+		WebResourceFetcherImpl onlineWrf = new WebResourceFetcherImpl();
+		WmfDumpFileManager dumpFileManager = new WmfDumpFileManager(
+				"wikidatawiki", dm, onlineWrf);
+
+		MwDumpFile dumpFile = dumpFileManager
+				.findMostRecentDump(DumpContentType.JSON);
+
+		assertTrue(dumpFile != null);
+		assertTrue("Dumpfile " + dumpFile
+						+ " should be online.",
+						dumpFile instanceof JsonOnlineDumpFile);
+	}
+
 	@Test
 	public void getAllCurrentDumps() throws IOException {
 		wrf.setWebResourceContentsFromResource(