Repository: komoot/photon (java)

Issue:
Add healthcheck endpoint
It would be nice to have a `/health` or `/status` endpoint for monitoring checks.

Pull request:
Add status API
This is a draft about the status API feature. Resolves #711 
I'm not sure about the Nominatim part of the code for import date, so if you have any idea of improvements feel free to ask.

Patch:
diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java
index b46bfef..bddbd33 100644
--- a/src/main/java/de/komoot/photon/App.java
+++ b/src/main/java/de/komoot/photon/App.java
@@ -14,6 +14,7 @@ import spark.Response;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.Date;
 
 import static spark.Spark.*;
 
@@ -56,8 +57,7 @@ public class App {
 
             if (args.isNominatimUpdate()) {
                 shutdownES = true;
-                final NominatimUpdater nominatimUpdater = setupNominatimUpdater(args, esServer);
-                nominatimUpdater.update();
+                startNominatimUpdate(args, esServer);
                 return;
             }
 
@@ -117,14 +117,15 @@ public class App {
      */
     private static void startNominatimImport(CommandLineArgs args, Server esServer) {
         DatabaseProperties dbProperties;
+        NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
+        Date importDate = nominatimConnector.getLastImportDate();
         try {
-            dbProperties = esServer.recreateIndex(args.getLanguages()); // clear out previous data
+            dbProperties = esServer.recreateIndex(args.getLanguages(), importDate); // clear out previous data
         } catch (IOException e) {
             throw new RuntimeException("Cannot setup index, elastic search config files not readable", e);
         }
 
         LOGGER.info("Starting import from nominatim to photon with languages: {}", String.join(",", dbProperties.getLanguages()));
-        NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
         nominatimConnector.setImporter(esServer.createImporter(dbProperties.getLanguages(), args.getExtraTags()));
         nominatimConnector.readEntireDatabase(args.getCountryCodes());
 
@@ -136,6 +137,22 @@ public class App {
         nominatimUpdater.initUpdates(args.getNominatimUpdateInit());
     }
 
+    private static void startNominatimUpdate(CommandLineArgs args, Server esServer) {
+        final NominatimUpdater nominatimUpdater = setupNominatimUpdater(args, esServer);
+        nominatimUpdater.update();
+
+        DatabaseProperties dbProperties = new DatabaseProperties();
+        esServer.loadFromDatabase(dbProperties);
+        NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
+        Date importDate = nominatimConnector.getLastImportDate();
+        dbProperties.setImportDate(importDate);
+        try {
+            esServer.saveToDatabase(dbProperties);
+        } catch (IOException e) {
+            throw new RuntimeException("Cannot setup index, elastic search config files not readable", e);
+        }
+    }
+
 
     /**
      * Prepare Nominatim updater.
@@ -174,6 +191,7 @@ public class App {
 
         // setup search API
         String[] langs = dbProperties.getLanguages();
+
         SearchHandler searchHandler = server.createSearchHandler(langs, args.getQueryTimeout());
         get("api", new SearchRequestHandler("api", searchHandler, langs, args.getDefaultLanguage()));
         get("api/", new SearchRequestHandler("api/", searchHandler, langs, args.getDefaultLanguage()));
@@ -181,12 +199,15 @@ public class App {
         ReverseHandler reverseHandler = server.createReverseHandler(args.getQueryTimeout());
         get("reverse", new ReverseSearchRequestHandler("reverse", reverseHandler, dbProperties.getLanguages(), args.getDefaultLanguage()));
         get("reverse/", new ReverseSearchRequestHandler("reverse/", reverseHandler, dbProperties.getLanguages(), args.getDefaultLanguage()));
+        
+        get("status", new StatusRequestHandler("status", server));
+        get("status/", new StatusRequestHandler("status/", server));
 
         if (args.isEnableUpdateApi()) {
             // setup update API
             final NominatimUpdater nominatimUpdater = setupNominatimUpdater(args, server);
             get("/nominatim-update", (Request request, Response response) -> {
-                new Thread(nominatimUpdater::update).start();
+                new Thread(()-> App.startNominatimUpdate(args, server)).start();
                 return "nominatim update started (more information in console output) ...";
             });
         }
diff --git a/src/main/java/de/komoot/photon/DatabaseProperties.java b/src/main/java/de/komoot/photon/DatabaseProperties.java
index 866b73b..773ee83 100644
--- a/src/main/java/de/komoot/photon/DatabaseProperties.java
+++ b/src/main/java/de/komoot/photon/DatabaseProperties.java
@@ -10,6 +10,11 @@ import java.util.*;
 public class DatabaseProperties {
     private String[] languages = null;
 
+    /**
+     * The OSM data date
+     */
+    private Date importDate;
+
     /**
      * Return the list of languages for which the database is configured.
      * @return
@@ -64,4 +69,13 @@ public class DatabaseProperties {
             languages = newLanguageList.toArray(new String[]{});
         }
     }
+
+    public Date getImportDate() {
+        return this.importDate;
+    }
+
+    public DatabaseProperties setImportDate(Date importDate) {
+        this.importDate = importDate;
+        return this;
+    }
 }
diff --git a/src/main/java/de/komoot/photon/StatusRequestHandler.java b/src/main/java/de/komoot/photon/StatusRequestHandler.java
new file mode 100644
index 0000000..36adec7
--- /dev/null
+++ b/src/main/java/de/komoot/photon/StatusRequestHandler.java
@@ -0,0 +1,36 @@
+package de.komoot.photon;
+
+import java.util.Date;
+
+import org.json.JSONObject;
+
+import de.komoot.photon.elasticsearch.Server;
+import spark.Request;
+import spark.Response;
+import spark.RouteImpl;
+
+public class StatusRequestHandler extends RouteImpl {
+    private Server server;
+
+    protected StatusRequestHandler(String path, Server server) {
+        super(path);
+        this.server = server;
+    }
+
+    @Override
+    public String handle(Request request, Response response) {
+        DatabaseProperties dbProperties = new DatabaseProperties();
+        server.loadFromDatabase(dbProperties);
+        String importDateStr = null;
+        if (dbProperties.getImportDate() instanceof Date) {
+            importDateStr = dbProperties.getImportDate().toInstant().toString();
+        }
+        
+        final JSONObject out = new JSONObject();
+        out.put("status", "Ok");
+        out.put("import_date", importDateStr);
+
+        return out.toString();
+    }
+    
+}
diff --git a/src/main/java/de/komoot/photon/elasticsearch/Server.java b/src/main/java/de/komoot/photon/elasticsearch/Server.java
index ca2d5a4..8cd0509 100644
--- a/src/main/java/de/komoot/photon/elasticsearch/Server.java
+++ b/src/main/java/de/komoot/photon/elasticsearch/Server.java
@@ -28,7 +28,9 @@ import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
+import java.time.Instant;
 import java.util.Collection;
+import java.util.Date;
 import java.util.LinkedList;
 import java.util.Map;
 
@@ -53,6 +55,7 @@ public class Server {
     private static final String BASE_FIELD = "document_properties";
     private static final String FIELD_VERSION = "database_version";
     private static final String FIELD_LANGUAGES = "indexed_languages";
+    private static final String FIELD_IMPORT_DATE = "import_date";
 
     private Node esNode;
 
@@ -169,14 +172,16 @@ public class Server {
 
     }
 
-    public DatabaseProperties recreateIndex(String[] languages) throws IOException {
+    public DatabaseProperties recreateIndex(String[] languages, Date importDate) throws IOException {
         deleteIndex();
 
         loadIndexSettings().createIndex(esClient, PhotonIndex.NAME);
 
         new IndexMapping().addLanguages(languages).putMapping(esClient, PhotonIndex.NAME, PhotonIndex.TYPE);
 
-        DatabaseProperties dbProperties = new DatabaseProperties().setLanguages(languages);
+        DatabaseProperties dbProperties = new DatabaseProperties()
+            .setLanguages(languages)
+            .setImportDate(importDate);
         saveToDatabase(dbProperties);
 
         return dbProperties;
@@ -223,6 +228,7 @@ public class Server {
         final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(BASE_FIELD)
                         .field(FIELD_VERSION, DATABASE_VERSION)
                         .field(FIELD_LANGUAGES, String.join(",", dbProperties.getLanguages()))
+                        .field(FIELD_IMPORT_DATE, dbProperties.getImportDate() instanceof Date ? dbProperties.getImportDate().toInstant() : null)
                         .endObject().endObject();
 
         esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE).
@@ -260,6 +266,9 @@ public class Server {
 
         String langString = properties.get(FIELD_LANGUAGES);
         dbProperties.setLanguages(langString == null ? null : langString.split(","));
+
+        String importDateString = properties.get(FIELD_IMPORT_DATE);
+        dbProperties.setImportDate(importDateString == null ? null : Date.from(Instant.parse(importDateString)));
     }
 
     public Importer createImporter(String[] languages, String[] extraTags) {
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 374e90c..72ba6cd 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -14,6 +14,7 @@ import org.springframework.jdbc.core.RowMapper;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -298,6 +299,19 @@ public class NominatimConnector {
         }
     }
 
+    public Date getLastImportDate() {
+        List<Date> importDates = template.query("SELECT lastimportdate FROM import_status ORDER BY lastimportdate DESC LIMIT 1", new RowMapper<Date>() {
+            public Date mapRow(ResultSet rs, int rowNum) throws SQLException {
+                return rs.getTimestamp("lastimportdate");
+            }
+        });
+        if (importDates.isEmpty()) {
+            return null;
+        }
+
+        return importDates.get(0);
+    }
+
     /**
      * Query Nominatim's address hierarchy to complete photon doc with missing data (like country, city, street, ...)
      *


Test patch:
diff --git a/src/test/java/de/komoot/photon/ApiIntegrationTest.java b/src/test/java/de/komoot/photon/ApiIntegrationTest.java
index 38a26f8..ec9b224 100644
--- a/src/test/java/de/komoot/photon/ApiIntegrationTest.java
+++ b/src/test/java/de/komoot/photon/ApiIntegrationTest.java
@@ -150,4 +150,17 @@ class ApiIntegrationTest extends ESBaseTester {
         assertEquals("city", properties.getString("osm_value"));
         assertEquals("berlin", properties.getString("name"));
     }
+
+    @Test
+    public void testApiStatus() throws Exception {
+        App.main(new String[]{"-cluster", TEST_CLUSTER_NAME, "-listen-port", Integer.toString(LISTEN_PORT), "-transport-addresses", "127.0.0.1"});
+        awaitInitialization();
+        DatabaseProperties prop = new DatabaseProperties();
+        getServer().loadFromDatabase(prop);
+        HttpURLConnection connection = (HttpURLConnection) new URL("http://127.0.0.1:" + port() + "/status").openConnection();
+        JSONObject json = new JSONObject(
+                new BufferedReader(new InputStreamReader(connection.getInputStream())).lines().collect(Collectors.joining("\n")));
+        assertEquals("Ok", json.getString("status"));
+        assertEquals(prop.getImportDate().toInstant().toString(), json.getString("import_date"));
+    }
 }
diff --git a/src/test/java/de/komoot/photon/ESBaseTester.java b/src/test/java/de/komoot/photon/ESBaseTester.java
index 4983054..3096dd8 100644
--- a/src/test/java/de/komoot/photon/ESBaseTester.java
+++ b/src/test/java/de/komoot/photon/ESBaseTester.java
@@ -12,6 +12,7 @@ import org.junit.jupiter.api.io.TempDir;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Collections;
+import java.util.Date;
 
 /**
  * Start an ES server with some test data that then can be queried in tests that extend this class
@@ -55,7 +56,7 @@ public class ESBaseTester {
     public void setUpES(Path test_directory, String... languages) throws IOException {
         server = new ElasticTestServer(test_directory.toString());
         server.start(TEST_CLUSTER_NAME, new String[]{});
-        server.recreateIndex(languages);
+        server.recreateIndex(languages, new Date());
         refresh();
     }
 
diff --git a/src/test/java/de/komoot/photon/elasticsearch/DatabasePropertiesTest.java b/src/test/java/de/komoot/photon/elasticsearch/DatabasePropertiesTest.java
index e4e74e6..ad51808 100644
--- a/src/test/java/de/komoot/photon/elasticsearch/DatabasePropertiesTest.java
+++ b/src/test/java/de/komoot/photon/elasticsearch/DatabasePropertiesTest.java
@@ -6,6 +6,8 @@ import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.*;
 
+import java.util.Date;
+
 /**
  * Tests for the database-global property store.
  */
@@ -19,7 +21,10 @@ class DatabasePropertiesTest extends ESBaseTester {
         DatabaseProperties prop = new DatabaseProperties();
 
         prop.setLanguages(new String[]{"en", "bg", "de"});
+        Date now = new Date();
+        prop.setImportDate(now);
         assertArrayEquals(new String[]{"en", "bg", "de"}, prop.getLanguages());
+        assertEquals(now, prop.getImportDate());
 
         prop.setLanguages(new String[]{"ru"});
         assertArrayEquals(new String[]{"ru"}, prop.getLanguages());
diff --git a/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java b/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java
index d3613f3..5cf6b54 100644
--- a/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java
+++ b/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java
@@ -5,6 +5,7 @@ import de.komoot.photon.ESBaseTester;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
+import java.util.Date;
 
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -16,13 +17,15 @@ class ServerTest extends ESBaseTester {
 
         DatabaseProperties prop = new DatabaseProperties();
         prop.setLanguages(new String[]{"en", "de", "fr"});
+        Date now = new Date();
+        prop.setImportDate(now);
         getServer().saveToDatabase(prop);
 
         prop = new DatabaseProperties();
         getServer().loadFromDatabase(prop);
 
         assertArrayEquals(new String[]{"en", "de", "fr"}, prop.getLanguages());
-
+        assertEquals(now, prop.getImportDate());
 
     }
 }
\ No newline at end of file
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
index 1df4cab..c29337f 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
@@ -12,6 +12,9 @@ import de.komoot.photon.nominatim.testdb.PlacexTestRow;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import static org.junit.jupiter.api.Assertions.*;
+
+import java.util.Date;
+
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase;
 import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
@@ -313,4 +316,14 @@ class NominatimConnectorDBTest {
 
         assertNull(importer.get(place).getCountryCode());
     }
+
+    @Test
+    public void testGetImportDate() {
+        Date importDate = connector.getLastImportDate();
+        assertNull(importDate);
+        
+        importDate = new Date();
+        jdbc.update("INSERT INTO import_status(lastimportdate, indexed) VALUES(?, ?)", importDate, true);
+        assertEquals(importDate, connector.getLastImportDate());
+    }
 }
\ No newline at end of file
diff --git a/src/test/resources/test-schema.sql b/src/test/resources/test-schema.sql
index b0c3ea4..652a9c6 100644
--- a/src/test/resources/test-schema.sql
+++ b/src/test/resources/test-schema.sql
@@ -71,3 +71,9 @@ CREATE TABLE photon_updates (
     operation TEXT,
     indexed_date TIMESTAMP
 );
+
+
+CREATE TABLE import_status (
+    lastimportdate timestamp with TIME ZONE NOT NULL,
+    indexed boolean
+);
\ No newline at end of file