Repository: gisaia/ARLAS-server (language: python)

Issue:
Support multi-collection in `partition-filter`



Discussion / hints:

Fix patch:
diff --git a/arlas-core/src/main/java/io/arlas/server/core/utils/ParamsParser.java b/arlas-core/src/main/java/io/arlas/server/core/utils/ParamsParser.java
index a5c789c5a..6abb9fc61 100644
--- a/arlas-core/src/main/java/io/arlas/server/core/utils/ParamsParser.java
+++ b/arlas-core/src/main/java/io/arlas/server/core/utils/ParamsParser.java
@@ -19,6 +19,8 @@
 
 package io.arlas.server.core.utils;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.arlas.server.core.exceptions.ArlasException;
 import io.arlas.server.core.exceptions.BadRequestException;
@@ -245,11 +247,16 @@ public static String getValidAggregationFormat(String aggFormat) {
         }
     }
 
-    public static Filter getFilter(String serializedFilter) throws InvalidParameterException {
+    public static Filter getFilter(CollectionReference collectionReference, String serializedFilter) throws InvalidParameterException {
         if (serializedFilter != null) {
             try {
-                return objectMapper.readValue(serializedFilter, Filter.class);
+                Map<String, Filter> pf = objectMapper.readValue(serializedFilter, new TypeReference<Map<String, Filter>>() {});
+                return pf.get(collectionReference.collectionName);
             } catch (IOException e) {
+                try {
+                    return objectMapper.readValue(serializedFilter, Filter.class);
+                } catch (JsonProcessingException ex) {
+                }
                 throw new InvalidParameterException(INVALID_FILTER + ": '" + serializedFilter + "'");
             }
         } else {
diff --git a/arlas-rest/src/main/java/io/arlas/server/rest/explore/aggregate/AggregateRESTService.java b/arlas-rest/src/main/java/io/arlas/server/rest/explore/aggregate/AggregateRESTService.java
index 2696f93b3..e05919c34 100644
--- a/arlas-rest/src/main/java/io/arlas/server/rest/explore/aggregate/AggregateRESTService.java
+++ b/arlas-rest/src/main/java/io/arlas/server/rest/explore/aggregate/AggregateRESTService.java
@@ -131,7 +131,7 @@ public Response aggregate(
         ColumnFilterUtil.assertRequestAllowed(columnFilter, collectionReference, aggregationsRequest);
 
         AggregationsRequest aggregationsRequestHeader = new AggregationsRequest();
-        aggregationsRequestHeader.filter = ParamsParser.getFilter(partitionFilter);
+        aggregationsRequestHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         request.basicRequest = aggregationsRequest;
         exploreService.setValidGeoFilters(collectionReference, aggregationsRequestHeader);
@@ -199,7 +199,7 @@ public Response aggregatePost(
         }
 
         AggregationsRequest aggregationsRequestHeader = new AggregationsRequest();
-        aggregationsRequestHeader.filter = ParamsParser.getFilter(partitionFilter);
+        aggregationsRequestHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         exploreService.setValidGeoFilters(collectionReference, aggregationsRequest);
         exploreService.setValidGeoFilters(collectionReference, aggregationsRequestHeader);
diff --git a/arlas-rest/src/main/java/io/arlas/server/rest/explore/aggregate/GeoAggregateRESTService.java b/arlas-rest/src/main/java/io/arlas/server/rest/explore/aggregate/GeoAggregateRESTService.java
index 470f606d9..2f842f936 100644
--- a/arlas-rest/src/main/java/io/arlas/server/rest/explore/aggregate/GeoAggregateRESTService.java
+++ b/arlas-rest/src/main/java/io/arlas/server/rest/explore/aggregate/GeoAggregateRESTService.java
@@ -587,7 +587,7 @@ public Response geoaggregatePost(
         }
 
         AggregationsRequest aggregationsRequestHeader = new AggregationsRequest();
-        aggregationsRequestHeader.filter = ParamsParser.getFilter(partitionFilter);
+        aggregationsRequestHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         exploreService.setValidGeoFilters(collectionReference, aggregationRequest);
         exploreService.setValidGeoFilters(collectionReference, aggregationsRequestHeader);
@@ -657,7 +657,7 @@ public Response shapeaggregatePost(
         }
 
         AggregationsRequest aggregationsRequestHeader = new AggregationsRequest();
-        aggregationsRequestHeader.filter = ParamsParser.getFilter(partitionFilter);
+        aggregationsRequestHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         exploreService.setValidGeoFilters(collectionReference, aggregationRequest);
         exploreService.setValidGeoFilters(collectionReference, aggregationsRequestHeader);
@@ -692,7 +692,7 @@ private MixedRequest getGeoaggregateRequest(CollectionReference collectionRefere
         ColumnFilterUtil.assertRequestAllowed(columnFilter, collectionReference, aggregationsRequest);
 
         AggregationsRequest aggregationsRequestHeader = new AggregationsRequest();
-        aggregationsRequestHeader.filter = ParamsParser.getFilter(partitionFilter);
+        aggregationsRequestHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         request.basicRequest = aggregationsRequest;
         exploreService.setValidGeoFilters(collectionReference, aggregationsRequestHeader);
diff --git a/arlas-rest/src/main/java/io/arlas/server/rest/explore/compute/ComputeRESTService.java b/arlas-rest/src/main/java/io/arlas/server/rest/explore/compute/ComputeRESTService.java
index deea96130..5418d59e1 100644
--- a/arlas-rest/src/main/java/io/arlas/server/rest/explore/compute/ComputeRESTService.java
+++ b/arlas-rest/src/main/java/io/arlas/server/rest/explore/compute/ComputeRESTService.java
@@ -127,7 +127,7 @@ public Response compute(
         ColumnFilterUtil.assertRequestAllowed(columnFilter, collectionReference, computationRequest);
 
         ComputationRequest computationRequestHeader = new ComputationRequest();
-        computationRequestHeader.filter = ParamsParser.getFilter(partitionFilter);
+        computationRequestHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         exploreService.setValidGeoFilters(collectionReference, computationRequestHeader);
         MixedRequest request = new MixedRequest();
         request.basicRequest = computationRequest;
@@ -191,7 +191,7 @@ public Response computePost(
             throw new NotFoundException(collection);
         }
         ComputationRequest computationRequestHeader = new ComputationRequest();
-        computationRequestHeader.filter = ParamsParser.getFilter(partitionFilter);
+        computationRequestHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         exploreService.setValidGeoFilters(collectionReference, computationRequest);
         exploreService.setValidGeoFilters(collectionReference, computationRequestHeader);
diff --git a/arlas-rest/src/main/java/io/arlas/server/rest/explore/count/CountRESTService.java b/arlas-rest/src/main/java/io/arlas/server/rest/explore/count/CountRESTService.java
index 23b0c58a8..0ae210ee6 100644
--- a/arlas-rest/src/main/java/io/arlas/server/rest/explore/count/CountRESTService.java
+++ b/arlas-rest/src/main/java/io/arlas/server/rest/explore/count/CountRESTService.java
@@ -114,7 +114,7 @@ public Response count(
         MixedRequest request = new MixedRequest();
         request.basicRequest = count;
         Count countHeader = new Count();
-        countHeader.filter = ParamsParser.getFilter(partitionfilter);
+        countHeader.filter = ParamsParser.getFilter(collectionReference, partitionfilter);
         exploreService.setValidGeoFilters(collectionReference, countHeader);
         request.headerRequest = countHeader;
         request.columnFilter = ColumnFilterUtil.getCollectionRelatedColumnFilter(columnFilter, collectionReference);
@@ -176,7 +176,7 @@ public Response countPost(
 
         request.basicRequest = count;
         Count countHeader = new Count();
-        countHeader.filter = ParamsParser.getFilter(partitionfilter);
+        countHeader.filter = ParamsParser.getFilter(collectionReference, partitionfilter);
         exploreService.setValidGeoFilters(collectionReference, countHeader);
         request.headerRequest = countHeader;
         request.columnFilter = ColumnFilterUtil.getCollectionRelatedColumnFilter(columnFilter, collectionReference);
diff --git a/arlas-rest/src/main/java/io/arlas/server/rest/explore/search/GeoSearchRESTService.java b/arlas-rest/src/main/java/io/arlas/server/rest/explore/search/GeoSearchRESTService.java
index 0659adbdf..91c9fcff0 100644
--- a/arlas-rest/src/main/java/io/arlas/server/rest/explore/search/GeoSearchRESTService.java
+++ b/arlas-rest/src/main/java/io/arlas/server/rest/explore/search/GeoSearchRESTService.java
@@ -490,7 +490,7 @@ public Response geosearchPost(
         CheckParams.checkReturnedGeometries(collectionReference, includes, excludes, search.returned_geometries);
 
         Search searchHeader = new Search();
-        searchHeader.filter = ParamsParser.getFilter(partitionFilter);
+        searchHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
 
         exploreService.setValidGeoFilters(collectionReference, search);
         exploreService.setValidGeoFilters(collectionReference, searchHeader);
@@ -564,7 +564,7 @@ public Response shapesearchPost(
         CheckParams.checkReturnedGeometries(collectionReference, includes, excludes, search.returned_geometries);
 
         Search searchHeader = new Search();
-        searchHeader.filter = ParamsParser.getFilter(partitionFilter);
+        searchHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
 
         exploreService.setValidGeoFilters(collectionReference, search);
         exploreService.setValidGeoFilters(collectionReference, searchHeader);
@@ -610,7 +610,7 @@ private Response geosearch(CollectionReference collectionReference, Filter filte
         search.projection = ParamsParser.enrichIncludes(search.projection, returned_geometries);
 
         Search searchHeader = new Search();
-        searchHeader.filter = ParamsParser.getFilter(partitionFilter);
+        searchHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         request.basicRequest = search;
         exploreService.setValidGeoFilters(collectionReference, searchHeader);
diff --git a/arlas-rest/src/main/java/io/arlas/server/rest/explore/search/SearchRESTService.java b/arlas-rest/src/main/java/io/arlas/server/rest/explore/search/SearchRESTService.java
index 278001add..d09baa2b4 100644
--- a/arlas-rest/src/main/java/io/arlas/server/rest/explore/search/SearchRESTService.java
+++ b/arlas-rest/src/main/java/io/arlas/server/rest/explore/search/SearchRESTService.java
@@ -185,7 +185,7 @@ public Response search(
         search.projection = ParamsParser.enrichIncludes(search.projection, returned_geometries);
 
         Search searchHeader = new Search();
-        searchHeader.filter = ParamsParser.getFilter(partitionFilter);
+        searchHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         request.basicRequest = search;
         exploreService.setValidGeoFilters(collectionReference, searchHeader);
@@ -252,7 +252,7 @@ public Response searchPost(
         String excludes = search.projection != null ? search.projection.excludes : null;
 
         Search searchHeader = new Search();
-        searchHeader.filter = ParamsParser.getFilter(partitionFilter);
+        searchHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
 
         exploreService.setValidGeoFilters(collectionReference, search);
         exploreService.setValidGeoFilters(collectionReference, searchHeader);
diff --git a/arlas-rest/src/main/java/io/arlas/server/rest/plugins/eo/TileRESTService.java b/arlas-rest/src/main/java/io/arlas/server/rest/plugins/eo/TileRESTService.java
index 06a160469..8a9090c27 100644
--- a/arlas-rest/src/main/java/io/arlas/server/rest/plugins/eo/TileRESTService.java
+++ b/arlas-rest/src/main/java/io/arlas/server/rest/plugins/eo/TileRESTService.java
@@ -187,7 +187,7 @@ public Response tiledgeosearch(
         ColumnFilterUtil.assertRequestAllowed(columnFilter, collectionReference, search);
 
         Search searchHeader = new Search();
-        searchHeader.filter = ParamsParser.getFilter(partitionFilter);
+        searchHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         request.basicRequest = search;
         request.headerRequest = searchHeader;
diff --git a/docs/arlas-api-exploration.md b/docs/arlas-api-exploration.md
index a2a9fedc4..378319ecf 100644
--- a/docs/arlas-api-exploration.md
+++ b/docs/arlas-api-exploration.md
@@ -276,7 +276,10 @@ In the case of `lt`, `gt`, `lte`, `gte`, `range` operations that are applied on
 
 `filter` part can also be passed in request header `partition-filter` as a serialized json for partitioning concerns.
 
-> Example: `curl --header "partition-filter: {f":[[{"field":"city","op":"eq","value":"Bordeaux"}]]}" https://api.gisaia.com/demo/arlas/explore/cities/_count`
+> Example: `curl --header "partition-filter: {"f":[[{"field":"city","op":"eq","value":"Bordeaux"}]]}" https://api.gisaia.com/demo/arlas/explore/cities/_count`
+
+When dealing with multi collections, filter can also be specified in a JSON map <collection name, filter>, e.g.:
+> Example: `curl --header "partition-filter: {"mycollection1": {"f":[[{"field":"city","op":"eq","value":"Bordeaux"}]]}}" https://api.gisaia.com/demo/arlas/explore/cities/_count`
 
 #### Column filtering
 
diff --git a/ogc-wfs/src/main/java/io/arlas/server/ogc/wfs/services/ElasticWFSToolService.java b/ogc-wfs/src/main/java/io/arlas/server/ogc/wfs/services/ElasticWFSToolService.java
index 5a791abfc..7be019d94 100644
--- a/ogc-wfs/src/main/java/io/arlas/server/ogc/wfs/services/ElasticWFSToolService.java
+++ b/ogc-wfs/src/main/java/io/arlas/server/ogc/wfs/services/ElasticWFSToolService.java
@@ -156,7 +156,7 @@ private void buildWFSQuery(WFSRequestType requestType, String id, String bbox, S
             buildStoredQueryIdQuery(id, storedquery_id, requestType, collectionReference);
         }
         if (partitionFilter != null) {
-            addPartitionFilter(fluidSearch, partitionFilter);
+            addPartitionFilter(collectionReference, fluidSearch, partitionFilter);
         }
     }
 
@@ -193,8 +193,8 @@ private void buildStoredQueryIdQuery(String id, String storedquery_id, WFSReques
         }
     }
 
-    private void addPartitionFilter(ElasticFluidSearch fluidSearch, String partitionFilter) throws ArlasException {
-        Filter headerFilter = ParamsParser.getFilter(partitionFilter);
+    private void addPartitionFilter(CollectionReference collectionReference, ElasticFluidSearch fluidSearch, String partitionFilter) throws ArlasException {
+        Filter headerFilter = ParamsParser.getFilter(collectionReference, partitionFilter);
         exploreServices.applyFilter(headerFilter, fluidSearch);
         wfsQuery.filter(fluidSearch.getBoolQueryBuilder());
     }
diff --git a/stac-api/src/main/java/io/arlas/server/stac/api/StacRESTService.java b/stac-api/src/main/java/io/arlas/server/stac/api/StacRESTService.java
index a27c209ea..4eaa348a6 100644
--- a/stac-api/src/main/java/io/arlas/server/stac/api/StacRESTService.java
+++ b/stac-api/src/main/java/io/arlas/server/stac/api/StacRESTService.java
@@ -232,7 +232,7 @@ protected StacFeatureCollection getStacFeatureCollection(CollectionReference col
         ColumnFilterUtil.assertRequestAllowed(columnFilter, collectionReference, search);
 
         Search searchHeader = new Search();
-        searchHeader.filter = ParamsParser.getFilter(partitionFilter);
+        searchHeader.filter = ParamsParser.getFilter(collectionReference, partitionFilter);
         MixedRequest request = new MixedRequest();
         request.basicRequest = search;
         request.headerRequest = searchHeader;


Test patch:
diff --git a/arlas-tests/src/test/java/io/arlas/server/tests/rest/explore/AbstractFilteredTest.java b/arlas-tests/src/test/java/io/arlas/server/tests/rest/explore/AbstractFilteredTest.java
index e057b9dd2..e4a5f7d79 100644
--- a/arlas-tests/src/test/java/io/arlas/server/tests/rest/explore/AbstractFilteredTest.java
+++ b/arlas-tests/src/test/java/io/arlas/server/tests/rest/explore/AbstractFilteredTest.java
@@ -36,7 +36,10 @@
 import org.junit.Before;
 import org.junit.Test;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+
 import static org.hamcrest.Matchers.*;
 
 public abstract class AbstractFilteredTest extends AbstractTestWithCollection {
@@ -952,6 +955,62 @@ public void testMixedFilter() throws Exception {
         request.filter = new Filter();
     }
 
+    @Test
+    public void testMixedFilterMultiCollectionFormat() throws Exception {
+        // valid bbox from WFS OGC SPEC = lower longitude , lower latitude , upper longitude  , upper latitude
+        // valid bbox for ARLAS classic bbox = lat top,  long left,  lat bottom,  long right
+        request.filter.f = Arrays.asList(new MultiValueFilter<>(new Expression("params.job", OperatorEnum.eq, "Architect")),//"job:eq:Architect"
+                new MultiValueFilter<>(new Expression("params.startdate", OperatorEnum.range, "[1009799<2000000]")),
+                new MultiValueFilter<>(new Expression("geo_params.centroid", OperatorEnum.within, "-50,-50,50,50")),
+                new MultiValueFilter<>(new Expression("geo_params.geometry", OperatorEnum.within, "-30,-30,30,30")),
+                new MultiValueFilter<>(new Expression("geo_params.geometry", OperatorEnum.intersects, "POLYGON((-20 20, 20 20, 20 -20, -20 -20, -20 20))")));
+
+        Map<String, Filter> filterHeader = new HashMap<String, Filter>();
+
+        Filter filter = new Filter();
+        filter.f = Arrays.asList(new MultiValueFilter<>(new Expression("params.startdate", OperatorEnum.range, "[0<1009801]")),
+                new MultiValueFilter<>(new Expression("geo_params.centroid", OperatorEnum.notwithin, "20,-50,60,50")),
+                new MultiValueFilter<>(new Expression("geo_params.geometry", OperatorEnum.notwithin, "POLYGON((-50 50,-20 50, -20 -50, -50 -50,-50 50))")),
+                new MultiValueFilter<>(new Expression("geo_params.geometry", OperatorEnum.notintersects, "POLYGON((-30 -10,30 10, 30 -30, -30 -30,-30 -10))")));
+        filterHeader.put("geodata", filter);
+        handleComplexFilter(
+                givenFilterableRequestParams()
+                        .header("partition-filter", objectMapper.writeValueAsString(filterHeader))
+                        .header("column-filter", "params.job,params.city,params.country")
+                        .param("f", new Expression("params.job", OperatorEnum.eq, "Architect").toString())
+                        .param("f", new Expression("params.startdate", OperatorEnum.range, "[1009799<2000000]").toString())
+                        .param("f", request.filter.f.get(2).get(0).toString())
+                        .param("f", request.filter.f.get(3).get(0).toString())
+                        .param("f", request.filter.f.get(4).get(0).toString())
+                        .when().get(getUrlPath("geodata"))
+                        .then());
+
+        handleComplexFilter(
+                post(
+                        request,
+                        "partition-filter",
+                        objectMapper.writeValueAsString(filterHeader),
+                        "column-filter",
+                        "params.job,params.city,params.country"));
+
+        filter.f = Arrays.asList(new MultiValueFilter<>(new Expression("params.job", OperatorEnum.eq, "Actor")));
+        handleNotMatchingRequest(
+                givenFilterableRequestParams()
+                        .header("partition-filter", objectMapper.writeValueAsString(filterHeader))
+                        .header("column-filter", "") //an empty column filter is not considered
+                        .param("f", (new Expression("params.job", OperatorEnum.eq, "Architect")).toString())
+                        .when().get(getUrlPath("geodata"))
+                        .then());
+        filterHeader.put("geodata", filter);
+        handleNotMatchingRequest(
+                givenFilterableRequestBody().body(request)
+                        .header("partition-filter", objectMapper.writeValueAsString(filterHeader))
+                        .header("column-filter", "")//an empty column filter is not considered
+                        .when().post(getUrlPath("geodata"))
+                        .then());
+        request.filter = new Filter();
+    }
+
     @Test
     public void testFieldFilterWithUnavailableColumns() throws Exception {
         request.filter = new Filter();