Repository: graphhopper/graphhopper (java)

Issue:
When using MMAP provide an option to force load the graph
Currently loading big graphs is possible and very fast with MMAP. The problem starts with the first requests: they are very slow.

To reduce this we could query the graph (and force the OS to load specific DataAccess objects from disk into memory) but it is unclear how to do that for arbitrary road data. Instead we can directly call `MappedByteBuffer.force()` for the important DataAccess objects like the ones of the BaseGraphStorage. We could even allow to specify a percentage for how many DataAccess objects we should call `force`. And have different values for base graph and for CH/LM (maybe even per-profile). 

Pull request description:
DataAccess config for type and load
Fixes #2416 and was discussed in #2432. Use in the config via:

```yml
graphhopper:

  graph.dataaccess.shortcuts_mycar: MMAP;30
  graph.dataaccess.nodes_ch_mycar: MMAP;30
  graph.dataaccess.shortcuts_car_rarely_used: MMAP;0
  graph.dataaccess.nodes_ch_car_rarely_used: MMAP;0

  # the default type:
  graph.dataaccess: RAM_STORE
  # so all other DataAccess objects like nodes, edges, geometry, location_index, string_index_keys, string_index_vals of the base graph get this default type
```

Patch:
diff --git a/config-example.yml b/config-example.yml
index a933e4855..fe95609f8 100644
--- a/config-example.yml
+++ b/config-example.yml
@@ -148,7 +148,7 @@ graphhopper:
 
 
   # configure the memory access, use RAM_STORE for well equipped servers (default and recommended)
-  graph.dataaccess: RAM_STORE
+  graph.dataaccess.default_type: RAM_STORE
 
 
   # will write way names in the preferred language (language code as defined in ISO 639-1 or ISO 639-2):
diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java
index 2f53e4585..bd7055198 100644
--- a/core/src/main/java/com/graphhopper/GraphHopper.java
+++ b/core/src/main/java/com/graphhopper/GraphHopper.java
@@ -90,7 +90,8 @@ public class GraphHopper {
     private EncodingManager encodingManager;
     private int defaultSegmentSize = -1;
     private String ghLocation = "";
-    private DAType dataAccessType = DAType.RAM_STORE;
+    private DAType dataAccessDefaultType = DAType.RAM_STORE;
+    private LinkedHashMap<String, String> dataAccessConfig = new LinkedHashMap<>();
     private boolean sortGraph = false;
     private boolean elevation = false;
     private LockFactory lockFactory = new NativeFSLockFactory();
@@ -177,9 +178,9 @@ public class GraphHopper {
     public GraphHopper setStoreOnFlush(boolean storeOnFlush) {
         ensureNotLoaded();
         if (storeOnFlush)
-            dataAccessType = DAType.RAM_STORE;
+            dataAccessDefaultType = DAType.RAM_STORE;
         else
-            dataAccessType = DAType.RAM;
+            dataAccessDefaultType = DAType.RAM;
         return this;
     }
 
@@ -421,8 +422,14 @@ public class GraphHopper {
 
         defaultSegmentSize = ghConfig.getInt("graph.dataaccess.segment_size", defaultSegmentSize);
 
-        String graphDATypeStr = ghConfig.getString("graph.dataaccess", "RAM_STORE");
-        dataAccessType = DAType.fromString(graphDATypeStr);
+        String daTypeString = ghConfig.getString("graph.dataaccess.default_type", ghConfig.getString("graph.dataaccess", "RAM_STORE"));
+        dataAccessDefaultType = DAType.fromString(daTypeString);
+        for (Map.Entry<String, Object> entry : ghConfig.asPMap().toMap().entrySet()) {
+            if (entry.getKey().startsWith("graph.dataaccess.type."))
+                dataAccessConfig.put(entry.getKey().substring("graph.dataaccess.type.".length()), entry.getValue().toString());
+            if (entry.getKey().startsWith("graph.dataaccess.mmap.preload."))
+                dataAccessConfig.put(entry.getKey().substring("graph.dataaccess.mmap.".length()), entry.getValue().toString());
+        }
 
         sortGraph = ghConfig.getBool("graph.do_sort", sortGraph);
         removeZipped = ghConfig.getBool("graph.remove_zipped", removeZipped);
@@ -727,17 +734,18 @@ public class GraphHopper {
             }
         }
 
-        if (!allowWrites && dataAccessType.isMMap())
-            dataAccessType = DAType.MMAP_RO;
+        if (!allowWrites && dataAccessDefaultType.isMMap())
+            dataAccessDefaultType = DAType.MMAP_RO;
         if (encodingManager == null) {
-            StorableProperties properties = new StorableProperties(new GHDirectory(ghLocation, dataAccessType));
+            StorableProperties properties = new StorableProperties(new GHDirectory(ghLocation, dataAccessDefaultType));
             encodingManager = properties.loadExisting()
                     ? EncodingManager.create(emBuilder, encodedValueFactory, flagEncoderFactory, properties)
                     : buildEncodingManager(new GraphHopperConfig());
         }
 
-        GHDirectory dir = new GHDirectory(ghLocation, dataAccessType);
-        ghStorage = new GraphHopperStorage(dir, encodingManager, hasElevation(), encodingManager.needsTurnCostsSupport(), defaultSegmentSize);
+        GHDirectory directory = new GHDirectory(ghLocation, dataAccessDefaultType);
+        directory.configure(dataAccessConfig);
+        ghStorage = new GraphHopperStorage(directory, encodingManager, hasElevation(), encodingManager.needsTurnCostsSupport(), defaultSegmentSize);
         checkProfilesConsistency();
 
         // todo: move this after base graph loading/import, just like for LM. there is no real reason we have to setup CH before
@@ -764,6 +772,7 @@ public class GraphHopper {
                 return false;
 
             postProcessing(false);
+            directory.loadMMap();
             setFullyLoaded();
             return true;
         } finally {
diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java
index 21b2f41b8..7aa905bc7 100644
--- a/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java
+++ b/core/src/main/java/com/graphhopper/storage/BaseGraphNodesAndEdges.java
@@ -63,8 +63,8 @@ class BaseGraphNodesAndEdges {
     private boolean frozen;
 
     public BaseGraphNodesAndEdges(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) {
-        nodes = dir.create("nodes", DAType.getPreferredInt(dir.getDefaultType()), segmentSize);
-        edges = dir.create("edges", DAType.getPreferredInt(dir.getDefaultType()), segmentSize);
+        nodes = dir.create("nodes", dir.getDefaultType("nodes", true), segmentSize);
+        edges = dir.create("edges", dir.getDefaultType("edges", true), segmentSize);
         this.intsForFlags = intsForFlags;
         this.withTurnCosts = withTurnCosts;
         this.withElevation = withElevation;
diff --git a/core/src/main/java/com/graphhopper/storage/CHStorage.java b/core/src/main/java/com/graphhopper/storage/CHStorage.java
index 92669c7d3..b1078a36f 100644
--- a/core/src/main/java/com/graphhopper/storage/CHStorage.java
+++ b/core/src/main/java/com/graphhopper/storage/CHStorage.java
@@ -68,8 +68,8 @@ public class CHStorage {
 
     public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased) {
         this.edgeBased = edgeBased;
-        this.nodesCH = dir.create("nodes_ch_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize);
-        this.shortcuts = dir.create("shortcuts_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize);
+        this.nodesCH = dir.create("nodes_ch_" + name, dir.getDefaultType("nodes_ch_" + name, true), segmentSize);
+        this.shortcuts = dir.create("shortcuts_" + name, dir.getDefaultType("shortcuts_" + name, true), segmentSize);
         // shortcuts are stored consecutively using this layout (the last two entries only exist for edge-based):
         // NODEA | NODEB | WEIGHT | SKIP_EDGE1 | SKIP_EDGE2 | S_ORIG_FIRST | S_ORIG_LAST
         S_NODEA = 0;
diff --git a/core/src/main/java/com/graphhopper/storage/DAType.java b/core/src/main/java/com/graphhopper/storage/DAType.java
index 88951a084..da75daf6b 100644
--- a/core/src/main/java/com/graphhopper/storage/DAType.java
+++ b/core/src/main/java/com/graphhopper/storage/DAType.java
@@ -87,15 +87,6 @@ public class DAType {
         return type;
     }
 
-    /**
-     * This method returns RAM_INT if the specified type is in-memory.
-     */
-    public static DAType getPreferredInt(DAType type) {
-        if (type.isInMemory())
-            return type.isStoring() ? RAM_INT_STORE : RAM_INT;
-        return type;
-    }
-
     /**
      * Memory mapped or purely in memory? default is HEAP
      */
diff --git a/core/src/main/java/com/graphhopper/storage/Directory.java b/core/src/main/java/com/graphhopper/storage/Directory.java
index 61c8bf7d6..03a3fc02f 100644
--- a/core/src/main/java/com/graphhopper/storage/Directory.java
+++ b/core/src/main/java/com/graphhopper/storage/Directory.java
@@ -56,6 +56,8 @@ public interface Directory {
      */
     DAType getDefaultType();
 
+    DAType getDefaultType(String dataAccess, boolean preferInts);
+
     /**
      * Removes all contained objects from the directory and releases its resources.
      */
diff --git a/core/src/main/java/com/graphhopper/storage/GHDirectory.java b/core/src/main/java/com/graphhopper/storage/GHDirectory.java
index a9990502d..df6582422 100644
--- a/core/src/main/java/com/graphhopper/storage/GHDirectory.java
+++ b/core/src/main/java/com/graphhopper/storage/GHDirectory.java
@@ -19,8 +19,11 @@ package com.graphhopper.storage;
 
 import java.io.File;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
+import static com.graphhopper.storage.DAType.RAM_INT;
+import static com.graphhopper.storage.DAType.RAM_INT_STORE;
 import static com.graphhopper.util.Helper.*;
 
 /**
@@ -30,11 +33,14 @@ import static com.graphhopper.util.Helper.*;
  */
 public class GHDirectory implements Directory {
     protected final String location;
-    private final DAType defaultType;
+    private final DAType typeFallback;
+    // first rule matches => LinkedHashMap
+    private final Map<String, DAType> defaultTypes = new LinkedHashMap<>();
+    private final Map<String, Integer> mmapPreloads = new LinkedHashMap<>();
     protected Map<String, DataAccess> map = new HashMap<>();
 
     public GHDirectory(String _location, DAType defaultType) {
-        this.defaultType = defaultType;
+        this.typeFallback = defaultType;
         if (isEmpty(_location))
             _location = new File("").getAbsolutePath();
 
@@ -47,14 +53,63 @@ public class GHDirectory implements Directory {
             throw new RuntimeException("file '" + dir + "' exists but is not a directory");
     }
 
+    /**
+     * Configure the DAType (specified by the value) of a single DataAccess object (specified by the key). For "MMAP" you
+     * can prepend "preload." to the name and specify a percentage which preloads the DataAccess into physical memory of
+     * the specified percentage (only applied for load, not for import).
+     * As keys can be patterns the order is important and the LinkedHashMap is forced as type.
+     */
+    public void configure(LinkedHashMap<String, String> config) {
+        for (Map.Entry<String, String> kv : config.entrySet()) {
+            String value = kv.getValue().trim();
+            if (kv.getKey().startsWith("preload."))
+                try {
+                    String pattern = kv.getKey().substring("preload.".length());
+                    mmapPreloads.put(pattern, Integer.parseInt(value));
+                } catch (NumberFormatException ex) {
+                    throw new IllegalArgumentException("DataAccess " + kv.getKey() + " has an incorrect preload value: " + value);
+                }
+            else {
+                String pattern = kv.getKey();
+                defaultTypes.put(pattern, DAType.fromString(value));
+            }
+        }
+    }
+
+    /**
+     * Returns the preload value or 0 if no patterns match.
+     * See {@link #configure(LinkedHashMap)}
+     */
+    int getPreload(String name) {
+        for (Map.Entry<String, Integer> entry : mmapPreloads.entrySet())
+            if (name.matches(entry.getKey())) return entry.getValue();
+        return 0;
+    }
+
+    public void loadMMap() {
+        for (DataAccess da : map.values()) {
+            if (!(da instanceof MMapDataAccess))
+                continue;
+            int preload = getPreload(da.getName());
+            if (preload > 0)
+                ((MMapDataAccess) da).load(preload);
+        }
+    }
+
     @Override
     public DataAccess create(String name) {
-        return create(name, defaultType);
+        return create(name, getDefault(name, typeFallback));
     }
 
     @Override
     public DataAccess create(String name, int segmentSize) {
-        return create(name, defaultType, segmentSize);
+        return create(name, getDefault(name, typeFallback), segmentSize);
+    }
+
+    private DAType getDefault(String name, DAType typeFallback) {
+        for (Map.Entry<String, DAType> entry : defaultTypes.entrySet())
+            if (name.matches(entry.getKey())) return entry.getValue();
+        return typeFallback;
     }
 
     @Override
@@ -127,11 +182,22 @@ public class GHDirectory implements Directory {
 
     @Override
     public DAType getDefaultType() {
-        return defaultType;
+        return typeFallback;
+    }
+
+    /**
+     * This method returns the default DAType of the specified DataAccess (as string). If preferInts is true then this
+     * method returns e.g. RAM_INT if the type of the specified DataAccess is RAM.
+     */
+    public DAType getDefaultType(String dataAccess, boolean preferInts) {
+        DAType type = getDefault(dataAccess, typeFallback);
+        if (preferInts && type.isInMemory())
+            return type.isStoring() ? RAM_INT_STORE : RAM_INT;
+        return type;
     }
 
     public boolean isStoring() {
-        return defaultType.isStoring();
+        return typeFallback.isStoring();
     }
 
     @Override
diff --git a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java
index 2dace1973..6424f88ee 100644
--- a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java
+++ b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java
@@ -307,6 +307,18 @@ public final class MMapDataAccess extends AbstractDataAccess {
         }
     }
 
+    /**
+     * Load memory mapped files into physical memory.
+     */
+    public void load(int percentage) {
+        if (percentage < 0 || percentage > 100)
+            throw new IllegalArgumentException("Percentage for MMapDataAccess.load for " + getName() + " must be in [0,100] but was " + percentage);
+        int max = Math.round(segments.size() * percentage / 100f);
+        for (int i = 0; i < max; i++) {
+            segments.get(i).load();
+        }
+    }
+
     @Override
     public void close() {
         super.close();
diff --git a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java
index ce687f468..554375a6e 100644
--- a/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java
+++ b/core/src/main/java/com/graphhopper/storage/index/LineIntIndex.java
@@ -21,7 +21,6 @@ package com.graphhopper.storage.index;
 import com.carrotsearch.hppc.IntArrayList;
 import com.carrotsearch.hppc.IntHashSet;
 import com.graphhopper.geohash.SpatialKeyAlgo;
-import com.graphhopper.storage.DAType;
 import com.graphhopper.storage.DataAccess;
 import com.graphhopper.storage.Directory;
 import com.graphhopper.util.Constants;
@@ -49,7 +48,7 @@ public class LineIntIndex {
 
     public LineIntIndex(BBox bBox, Directory dir, String name) {
         this.bounds = bBox;
-        this.dataAccess = dir.create(name, DAType.getPreferredInt(dir.getDefaultType()));
+        this.dataAccess = dir.create(name, dir.getDefaultType(name, true));
     }
 
     public boolean loadExisting() {
diff --git a/web-api/src/main/java/com/graphhopper/util/PMap.java b/web-api/src/main/java/com/graphhopper/util/PMap.java
index 9a463f673..639d67371 100644
--- a/web-api/src/main/java/com/graphhopper/util/PMap.java
+++ b/web-api/src/main/java/com/graphhopper/util/PMap.java
@@ -27,7 +27,7 @@ import java.util.Map;
  * @author Peter Karich
  */
 public class PMap {
-    private final Map<String, Object> map;
+    private final LinkedHashMap<String, Object> map;
 
     public PMap() {
         this(5);


Test patch:
diff --git a/core/src/test/java/com/graphhopper/storage/GHDirectoryTest.java b/core/src/test/java/com/graphhopper/storage/GHDirectoryTest.java
new file mode 100644
index 000000000..da00d8824
--- /dev/null
+++ b/core/src/test/java/com/graphhopper/storage/GHDirectoryTest.java
@@ -0,0 +1,36 @@
+package com.graphhopper.storage;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.LinkedHashMap;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class GHDirectoryTest {
+
+    @Test
+    public void testConfigure() {
+        GHDirectory dir = new GHDirectory("", DAType.RAM_STORE);
+        LinkedHashMap<String, String> map = new LinkedHashMap<>();
+        map.put("nodes", "MMAP");
+        dir.configure(map);
+        assertEquals(DAType.MMAP, dir.getDefaultType("nodes", true));
+
+        // first rule wins
+        map.put("preload.nodes", "10");
+        map.put("preload.nodes.*", "100");
+        dir.configure(map);
+        assertEquals(10, dir.getPreload("nodes"));
+    }
+
+    @Test
+    public void testPatternMatching() {
+        GHDirectory dir = new GHDirectory("", DAType.RAM_STORE);
+        LinkedHashMap<String, String> map = new LinkedHashMap<>();
+        map.put("nodes_ch.*", "MMAP");
+        dir.configure(map);
+        assertEquals(DAType.RAM_STORE, dir.getDefaultType("nodes", false));
+        assertEquals(DAType.MMAP, dir.getDefaultType("nodes_ch_car", false));
+    }
+
+}
\ No newline at end of file