Repository: ad-freiburg/qlever
Pull request #1941: Make `<libspatialjoin>` aware of WKT geometries from `LocalVocab` 
Author: login: patrickbr
Created: 2025-04-07T12:30:06Z | Merged: 2025-04-08T23:02:00Z

When doing spatial joins with `<libspatialjoin>`, the `LocalVocab`s of the two input tables were ignored so far. In particular, geometries that were not part of the original index were therefore ignored. This is now fixed. In particular, fixes #1939

On the side, drop `--ffast-math` from the compilation options of `libspatialjoin` to fix some warnings in the `fmt` library and `Geo.h`, in which `infinity` was still used


Diff:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index da61ef9d61..38b23d718b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -111,7 +111,7 @@ FetchContent_Declare(
 FetchContent_Declare(
         spatialjoin
         GIT_REPOSITORY https://github.com/ad-freiburg/spatialjoin
-        GIT_TAG 21be46ab71ecbc8400daa8c5a19f47853dea0ac8
+        GIT_TAG c1e6e3dfa918fa1d6ed0a25210156dc914392e0f
 )
 # disable bzip2 and zlib support in spatialjoin, we don't need it
 add_definitions("-DSPATIALJOIN_NO_BZIP2=True")
diff --git a/src/engine/SpatialJoinAlgorithms.cpp b/src/engine/SpatialJoinAlgorithms.cpp
index 7cb90321ef..51fa983992 100644
--- a/src/engine/SpatialJoinAlgorithms.cpp
+++ b/src/engine/SpatialJoinAlgorithms.cpp
@@ -52,6 +52,13 @@ util::geo::I32Box SpatialJoinAlgorithms::libspatialjoinParse(
       const auto& p = id.getGeoPoint();
       parser.parsePoint(util::geo::DPoint(p.getLng(), p.getLat()), row,
                         leftOrRightSide);
+    } else if (id.getDatatype() == Datatype::LocalVocabIndex) {
+      const auto& literalOrIri = *id.getLocalVocabIndex();
+      if (literalOrIri.isLiteral()) {
+        const auto& wkt =
+            asStringViewUnsafe(literalOrIri.getLiteral().getContent());
+        parser.parseWKT(wkt, row, leftOrRightSide);
+      }
     }
   }
 
diff --git a/test/engine/SpatialJoinTest.cpp b/test/engine/SpatialJoinTest.cpp
index ae7c406a6b..53884fd016 100644
--- a/test/engine/SpatialJoinTest.cpp
+++ b/test/engine/SpatialJoinTest.cpp
@@ -27,10 +27,12 @@
 #include "engine/Join.h"
 #include "engine/QueryExecutionContext.h"
 #include "engine/QueryExecutionTree.h"
+#include "engine/QueryPlanner.h"
 #include "engine/SpatialJoin.h"
 #include "engine/VariableToColumnMap.h"
 #include "global/Constants.h"
 #include "gmock/gmock.h"
+#include "parser/SparqlParser.h"
 #include "parser/data/Variable.h"
 
 namespace {  // anonymous namespace to avoid linker problems
@@ -221,6 +223,42 @@ using VarColTestSuiteParam = std::tuple<bool, bool, bool, bool>;
 using V = Variable;
 using VarToColVec = std::vector<std::pair<V, ColumnIndexAndTypeInfo>>;
 
+// Helper function to create a spatial join from VALUEs
+std::shared_ptr<SpatialJoin> makeSpatialJoinFromValues(
+    QueryExecutionContext* qec, PayloadVariables pv = PayloadVariables::all(),
+    SpatialJoinAlgorithm alg = SPATIAL_JOIN_DEFAULT_ALGORITHM) {
+  const auto sharedHandle =
+      std::make_shared<ad_utility::CancellationHandle<>>();
+  // also include some garbage input geometries
+  auto pqLeft = SparqlParser::parseQuery(
+      "PREFIX geo: <http://www.opengis.net/ont/geosparql#>\nSELECT ?a {VALUES "
+      "(?a) {(\"POLYGON((8.529 47.375, 8.549 47.375, 8.549 47.395, 8.529 "
+      "47.395, 8.529 47.375))\"^^geo:wktLiteral) (\"garbage\") (5) (<>)}}");
+  auto pqRight = SparqlParser::parseQuery(
+      "PREFIX geo: <http://www.opengis.net/ont/geosparql#>\nSELECT ?b {VALUES "
+      "(?b) {(\"POINT(8.542 47.385)\"^^geo:wktLiteral)}}");
+  QueryPlanner qp{qec, sharedHandle};
+
+  auto leftChild =
+      std::make_shared<QueryExecutionTree>(qp.createExecutionTree(pqLeft));
+  auto rightChild =
+      std::make_shared<QueryExecutionTree>(qp.createExecutionTree(pqRight));
+
+  std::shared_ptr<QueryExecutionTree> spatialJoinOperation =
+      ad_utility::makeExecutionTree<SpatialJoin>(
+          qec,
+          SpatialJoinConfiguration{MaxDistanceConfig{0}, Variable{"?a"},
+                                   Variable{"?b"}, std::nullopt, pv, alg,
+                                   SpatialJoinType::INTERSECTS},
+          std::nullopt, std::nullopt);
+  std::shared_ptr<Operation> op = spatialJoinOperation->getRootOperation();
+  SpatialJoin* spatialJoin = static_cast<SpatialJoin*>(op.get());
+  auto spJoin1 = spatialJoin->addChild(leftChild, Variable{"?a"});
+  spatialJoin = static_cast<SpatialJoin*>(spJoin1.get());
+  auto spJoin2 = spatialJoin->addChild(rightChild, Variable{"?b"});
+  return spJoin2;
+}
+
 class SpatialJoinVarColParamTest
     : public ::testing::TestWithParam<VarColTestSuiteParam> {
  public:
@@ -370,7 +408,7 @@ class SpatialJoinVarColParamTest
     }
   }
 
-  // TODO: comment (avoid redundancy with comment below).
+  // Test spatial join on contains
   void testGetResultWidthOrVariableToColumnMapSpatialJoinContains(
       VarColTestSuiteParam parameters) {
     auto [leftSideBigChild, rightSideBigChild, addLeftChildFirst,
@@ -608,6 +646,18 @@ TEST_P(SpatialJoinVarColParamTest, variableToColumnMapLibspatialjoinContains) {
   testGetResultWidthOrVariableToColumnMapSpatialJoinContains(GetParam());
 }
 
+// Test a spatial join with VALUES as both children
+TEST(SpatialJoinVarColParamTest, testLibspatialjoinFromvalues) {
+  auto qec = buildNonSelfJoinDataset();
+
+  auto spJoin2 = makeSpatialJoinFromValues(
+      qec, PayloadVariables::all(), SpatialJoinAlgorithm::LIBSPATIALJOIN);
+  auto spatialJoin = static_cast<SpatialJoin*>(spJoin2.get());
+
+  auto resultTable = spatialJoin->computeResult(false);
+  ASSERT_EQ(resultTable.idTable().numRows(), 1);
+}
+
 TEST_P(SpatialJoinVarColParamTest, payloadVariables) {
   testPayloadVariablesVarToColMap(GetParam());
 }


Reviews:
user: login: joka921
body: Some suggestions, but overall this looks nice.
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-04-07T17:12:35Z
user: login: patrickbr
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-08T09:18:08Z
user: login: joka921
body: Thank you very much, this looks clean and tested now.
@hannahbast  you can add this PR to your internal merge queue.
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-08T16:38:51Z

Review comments:
diff_hunk: @@ -52,6 +53,15 @@ util::geo::I32Box SpatialJoinAlgorithms::libspatialjoinParse(
       const auto& p = id.getGeoPoint();
       parser.parsePoint(util::geo::DPoint(p.getLng(), p.getLat()), row,
                         leftOrRightSide);
+    } else if (id.getDatatype() == Datatype::LocalVocabIndex) {
+      const auto& literalOrIri = result->localVocab()
+                                     .getWord(id.getLocalVocabIndex())
+                                     .asLiteralOrIri();
path: src/engine/SpatialJoinAlgorithms.cpp
user: login: joka921
body: ```suggestion
      const auto& literalOrIri = *id.getLocalVocabIndex();
```
(We just encode raw pointers into the id, the local vocab is just for keeping the IDs alive).
created_at: 2025-04-07T17:04:50Z
author_association: MEMBER
diff_hunk: @@ -52,6 +53,15 @@ util::geo::I32Box SpatialJoinAlgorithms::libspatialjoinParse(
       const auto& p = id.getGeoPoint();
       parser.parsePoint(util::geo::DPoint(p.getLng(), p.getLat()), row,
                         leftOrRightSide);
+    } else if (id.getDatatype() == Datatype::LocalVocabIndex) {
+      const auto& literalOrIri = result->localVocab()
+                                     .getWord(id.getLocalVocabIndex())
+                                     .asLiteralOrIri();
+      if (literalOrIri.isLiteral()) {
+        const auto& wkt = std::string(
+            asStringViewUnsafe(literalOrIri.getLiteral().getContent()));
+        parser.parseWKT(wkt.c_str(), row, leftOrRightSide);
path: src/engine/SpatialJoinAlgorithms.cpp
user: login: joka921
body: How hard is it to let the `parseWKT` function handle `string_view` natively to avoid the unneeded string copy here? At the end of the `content` there is either a `\0`, an `@` or a `^`, if that helps your internal parsing routine.
created_at: 2025-04-07T17:10:55Z
author_association: MEMBER
diff_hunk: @@ -52,6 +53,15 @@ util::geo::I32Box SpatialJoinAlgorithms::libspatialjoinParse(
       const auto& p = id.getGeoPoint();
       parser.parsePoint(util::geo::DPoint(p.getLng(), p.getLat()), row,
                         leftOrRightSide);
+    } else if (id.getDatatype() == Datatype::LocalVocabIndex) {
+      const auto& literalOrIri = result->localVocab()
+                                     .getWord(id.getLocalVocabIndex())
+                                     .asLiteralOrIri();
+      if (literalOrIri.isLiteral()) {
+        const auto& wkt = std::string(
+            asStringViewUnsafe(literalOrIri.getLiteral().getContent()));
+        parser.parseWKT(wkt.c_str(), row, leftOrRightSide);
+      }
path: src/engine/SpatialJoinAlgorithms.cpp
user: login: joka921
body: This code should be testable, 
you can just for example create a `VALUES`clause as one of the inputs that hold geoliterals that are not part of the original index.
created_at: 2025-04-07T17:12:19Z
author_association: MEMBER
diff_hunk: @@ -52,6 +53,15 @@ util::geo::I32Box SpatialJoinAlgorithms::libspatialjoinParse(
       const auto& p = id.getGeoPoint();
       parser.parsePoint(util::geo::DPoint(p.getLng(), p.getLat()), row,
                         leftOrRightSide);
+    } else if (id.getDatatype() == Datatype::LocalVocabIndex) {
+      const auto& literalOrIri = result->localVocab()
+                                     .getWord(id.getLocalVocabIndex())
+                                     .asLiteralOrIri();
+      if (literalOrIri.isLiteral()) {
+        const auto& wkt = std::string(
+            asStringViewUnsafe(literalOrIri.getLiteral().getContent()));
+        parser.parseWKT(wkt.c_str(), row, leftOrRightSide);
path: src/engine/SpatialJoinAlgorithms.cpp
user: login: patrickbr
body: I now added pseudo-native support: there is a least a `parseWKT()` method that accepts a `std::string_view`. This string view is immediately copied onto the job queue, however (in the `spatialjoin` tool, the WKT strings come directly from the file read buffer, so it is not guaranteed that they will live until a worker has touched the parse job, hence the copying).

But if these string views are guaranteed to live until the query has finished, we could theoretically avoid the copying here and store the `std::string_view` directly in the parse job. However, this would also not be trivial, as the parser heavily relies on C string functions (in particular `strchr`) internally, so it must be null terminated. I could replace them by `memchr` and keep track of the remaining length, but this is likely to be messy and will take some time.
created_at: 2025-04-08T09:18:07Z
author_association: MEMBER

PR comments:
user: login: codecov[bot]
created_at: 2025-04-07T13:51:43Z
body: ## [Codecov](https://app.codecov.io/gh/ad-freiburg/qlever/pull/1941?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ad-freiburg) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 90.72%. Comparing base [(`9686503`)](https://app.codecov.io/gh/ad-freiburg/qlever/commit/96865035c90afdd4207faed4c022fd3f70c249ab?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ad-freiburg) to head [(`1d7ed43`)](https://app.codecov.io/gh/ad-freiburg/qlever/commit/1d7ed432e8a13abeed9f1849e80a0ee2fcd5e21e?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ad-freiburg).
> Report is 3 commits behind head on master.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #1941      +/-   ##
==========================================
- Coverage   90.72%   90.72%   -0.01%     
==========================================
  Files         416      416              
  Lines       40838    40845       +7     
  Branches     4614     4616       +2     
==========================================
+ Hits        37050    37056       +6     
  Misses       2423     2423              
- Partials     1365     1366       +1     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/ad-freiburg/qlever/pull/1941?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ad-freiburg).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=ad-freiburg).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: sparql-conformance[bot]
created_at: 2025-04-08T15:32:02Z
body: 
### Conformance check passed ✅


 No test result changes.

Details: https://qlever.cs.uni-freiburg.de/sparql-conformance-ui?cur=1d7ed432e8a13abeed9f1849e80a0ee2fcd5e21e&prev=96865035c90afdd4207faed4c022fd3f70c249ab
author_association: NONE
user: login: sonarqubecloud[bot]
created_at: 2025-04-08T16:02:38Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=ad-freiburg_qlever&pullRequest=1941) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [1 New issue](https://sonarcloud.io/project/issues?id=ad-freiburg_qlever&pullRequest=1941&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=ad-freiburg_qlever&pullRequest=1941&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=ad-freiburg_qlever&pullRequest=1941&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=ad-freiburg_qlever&pullRequest=1941&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=ad-freiburg_qlever&pullRequest=1941&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=ad-freiburg_qlever&pullRequest=1941)


author_association: NONE