Repository: kestra-io/kestra (java)

Issue:
Filter don't work anymore
### Describe the issue

It's not possible to filter anymore on preview environment

https://postgres-oss.preview.dev.kestra.io/ui/namespaces/edit/sanitychecks/flows?filters%5Bnamespace%5D%5BEQUALS%5D=sanitychecks&filters%5Bq%5D%5B$eq%5D=sync&size=10&page=1

![image](https://github.com/user-attachments/assets/7410b41f-90f1-47a3-a04c-40acf44929be)


### Environment

- Kestra Version: develop


Pull request:
fix(): datetime filtering issues
close #7959


Patch:
diff --git a/core/src/main/java/io/kestra/core/runners/pebble/functions/KvFunction.java b/core/src/main/java/io/kestra/core/runners/pebble/functions/KvFunction.java
index cdf7a4092..60250baa7 100644
--- a/core/src/main/java/io/kestra/core/runners/pebble/functions/KvFunction.java
+++ b/core/src/main/java/io/kestra/core/runners/pebble/functions/KvFunction.java
@@ -1,5 +1,6 @@
 package io.kestra.core.runners.pebble.functions;
 
+import io.kestra.core.exceptions.ResourceExpiredException;
 import io.kestra.core.services.KVStoreService;
 import io.kestra.core.storages.kv.KVValue;
 import io.pebbletemplates.pebble.error.PebbleException;
@@ -8,6 +9,7 @@ import io.pebbletemplates.pebble.template.EvaluationContext;
 import io.pebbletemplates.pebble.template.PebbleTemplate;
 import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
+import java.io.IOException;
 import lombok.extern.slf4j.Slf4j;
 
 import java.util.List;
@@ -42,14 +44,15 @@ public class KvFunction implements Function {
         String flowNamespace = flow.get(NAMESPACE_ARG);
         String flowTenantId = flow.get("tenantId");
 
-        // we didn't check allowedNamespace here as it's checked in the kvStoreService itself
-        if (namespace == null) {
-            namespace = flowNamespace;
-        }
-
         Optional<KVValue> value;
         try {
-            value = kvStoreService.get(flowTenantId, namespace, flowNamespace).getValue(key);
+            if (namespace == null) {
+                namespace = flowNamespace;
+                value = getValueWithInheritance(flowNamespace, key, flowTenantId);
+            } else {
+                // we didn't check allowedNamespace here as it's checked in the kvStoreService itself
+                value = kvStoreService.get(flowTenantId, namespace, flowNamespace).getValue(key);
+            }
         } catch (Exception e) {
             throw new PebbleException(e, e.getMessage(), lineNumber, self.getName());
         }
@@ -61,6 +64,20 @@ public class KvFunction implements Function {
         return value.map(KVValue::value).orElse(null);
     }
 
+    private Optional<KVValue> getValueWithInheritance(String flowNamespace, String key, String tenantId)
+        throws IOException, ResourceExpiredException {
+        Optional<KVValue> value = Optional.empty();
+        String inheritedNamespace = flowNamespace;
+        while (value.isEmpty()) {
+            value = kvStoreService.get(tenantId, inheritedNamespace, flowNamespace).getValue(key);
+            if (!inheritedNamespace.contains(".")){
+                return value;
+            }
+            inheritedNamespace = inheritedNamespace.substring(0, inheritedNamespace.lastIndexOf('.'));
+        }
+        return value;
+    }
+
     protected String getKey(Map<String, Object> args, PebbleTemplate self, int lineNumber) {
         if (!args.containsKey(KEY_ARGS)) {
             throw new PebbleException(null, "The 'kv' function expects an argument 'key'.", lineNumber, self.getName());
diff --git a/core/src/main/java/io/kestra/plugin/core/kv/Get.java b/core/src/main/java/io/kestra/plugin/core/kv/Get.java
index b26ef4a28..f20e615f0 100644
--- a/core/src/main/java/io/kestra/plugin/core/kv/Get.java
+++ b/core/src/main/java/io/kestra/plugin/core/kv/Get.java
@@ -1,5 +1,6 @@
 package io.kestra.plugin.core.kv;
 
+import io.kestra.core.exceptions.ResourceExpiredException;
 import io.kestra.core.models.annotations.Example;
 import io.kestra.core.models.annotations.Plugin;
 import io.kestra.core.models.property.Property;
@@ -8,9 +9,12 @@ import io.kestra.core.models.tasks.Task;
 import io.kestra.core.runners.DefaultRunContext;
 import io.kestra.core.runners.RunContext;
 import io.kestra.core.services.FlowService;
+import io.kestra.core.services.KVStoreService;
 import io.kestra.core.storages.kv.KVValue;
 import io.swagger.v3.oas.annotations.media.Schema;
 import jakarta.validation.constraints.NotNull;
+import java.io.IOException;
+import java.util.Objects;
 import lombok.Builder;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
@@ -71,22 +75,42 @@ public class Get extends Task implements RunnableTask<Get.Output> {
     @Override
     public Output run(RunContext runContext) throws Exception {
         String renderedNamespace = runContext.render(this.namespace).as(String.class).orElse(null);
-
-        FlowService flowService = ((DefaultRunContext) runContext).getApplicationContext().getBean(FlowService.class);
-        flowService.checkAllowedNamespace(runContext.flowInfo().tenantId(), renderedNamespace, runContext.flowInfo().tenantId(), runContext.flowInfo().namespace());
-
+        String flowNamespace = runContext.flowInfo().namespace();
         String renderedKey = runContext.render(this.key).as(String.class).orElse(null);
 
-        Optional<KVValue> maybeValue = runContext.namespaceKv(renderedNamespace).getValue(renderedKey);
-        if (Boolean.TRUE.equals(runContext.render(this.errorOnMissing).as(Boolean.class).orElseThrow()) && maybeValue.isEmpty()) {
+        Optional<KVValue> value;
+        if (Objects.equals(renderedNamespace, flowNamespace)) {
+            value = getValueWithInheritance(runContext, flowNamespace, renderedKey);
+        } else {
+            FlowService flowService = ((DefaultRunContext) runContext).getApplicationContext().getBean(FlowService.class);
+            flowService.checkAllowedNamespace(runContext.flowInfo().tenantId(), renderedNamespace, runContext.flowInfo().tenantId(), runContext.flowInfo().namespace());
+            value =  runContext.namespaceKv(renderedNamespace).getValue(renderedKey);
+        }
+
+        if (Boolean.TRUE.equals(runContext.render(this.errorOnMissing).as(Boolean.class).orElseThrow()) && value.isEmpty()) {
             throw new NoSuchElementException("No value found for key '" + renderedKey + "' in namespace '" + renderedNamespace + "' and `errorOnMissing` is set to true");
         }
 
         return Output.builder()
-            .value(maybeValue.map(KVValue::value).orElse(null))
+            .value(value.map(KVValue::value).orElse(null))
             .build();
     }
 
+    private Optional<KVValue> getValueWithInheritance(RunContext runContext, String flowNamespace, String renderedKey)
+            throws IOException, ResourceExpiredException {
+        Optional<KVValue> value = Optional.empty();
+        KVStoreService kvStoreService = ((DefaultRunContext) runContext).getApplicationContext().getBean(KVStoreService.class);
+        String inheritedNamespace = flowNamespace;
+        while (value.isEmpty()) {
+            value = kvStoreService.get(runContext.flowInfo().tenantId(), inheritedNamespace, flowNamespace).getValue(renderedKey);
+            if (!inheritedNamespace.contains(".")){
+                return value;
+            }
+            inheritedNamespace = inheritedNamespace.substring(0, inheritedNamespace.lastIndexOf('.'));
+        }
+        return value;
+    }
+
     @Builder
     @Getter
     public static class Output implements io.kestra.core.models.tasks.Output {
diff --git a/jdbc-h2/src/main/java/io/kestra/repository/h2/H2ExecutionRepository.java b/jdbc-h2/src/main/java/io/kestra/repository/h2/H2ExecutionRepository.java
index a18607951..ae4aec051 100644
--- a/jdbc-h2/src/main/java/io/kestra/repository/h2/H2ExecutionRepository.java
+++ b/jdbc-h2/src/main/java/io/kestra/repository/h2/H2ExecutionRepository.java
@@ -55,4 +55,4 @@ public class H2ExecutionRepository extends AbstractJdbcExecutionRepository {
                 throw new IllegalArgumentException("Unsupported GroupType: " + groupType);
         }
     }
-}
+}
\ No newline at end of file
diff --git a/jdbc-h2/src/main/java/io/kestra/repository/h2/H2ExecutionRepositoryService.java b/jdbc-h2/src/main/java/io/kestra/repository/h2/H2ExecutionRepositoryService.java
index f1accf380..77586d5d0 100644
--- a/jdbc-h2/src/main/java/io/kestra/repository/h2/H2ExecutionRepositoryService.java
+++ b/jdbc-h2/src/main/java/io/kestra/repository/h2/H2ExecutionRepositoryService.java
@@ -11,8 +11,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-import static io.kestra.core.models.QueryFilter.Op.EQUALS;
-
 public abstract class H2ExecutionRepositoryService {
     public static Condition findCondition(AbstractJdbcRepository<Execution> jdbcRepository, String query, Map<String, String> labels) {
         List<Condition> conditions = new ArrayList<>();
diff --git a/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcExecutionRepository.java b/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcExecutionRepository.java
index 34255ea52..1e2fb98a8 100644
--- a/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcExecutionRepository.java
+++ b/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcExecutionRepository.java
@@ -41,10 +41,7 @@ import org.jooq.impl.DSL;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.FluxSink;
 
-import java.time.Duration;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
+import java.time.*;
 import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
@@ -300,7 +297,7 @@ public abstract class AbstractJdbcExecutionRepository extends AbstractJdbcReposi
                 } else if (field.equals(QueryFilter.Field.LABELS) && value instanceof Map<?, ?> labels)
                     select = select.and(findCondition(labels, operation));
                 else
-                    select = getConditionOnField(select, field, value, operation, "start_date");
+                    select = getConditionOnField(select, field, value, operation, "\"start_date\"");
             }
 
         return select;
diff --git a/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcRepository.java b/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcRepository.java
index 131be4b5a..8dcecf82b 100644
--- a/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcRepository.java
+++ b/jdbc/src/main/java/io/kestra/jdbc/repository/AbstractJdbcRepository.java
@@ -424,7 +424,9 @@ public abstract class AbstractJdbcRepository {
     ) {
         switch (operation) {
             case LESS_THAN -> select = select.and(DSL.field(fieldName).lessThan(dateTime));
+            case LESS_THAN_OR_EQUAL_TO -> select = select.and(DSL.field(fieldName).lessOrEqual(dateTime));
             case GREATER_THAN -> select = select.and(DSL.field(fieldName).greaterThan(dateTime));
+            case GREATER_THAN_OR_EQUAL_TO -> select = select.and(DSL.field(fieldName).greaterOrEqual(dateTime));
             case EQUALS -> select = select.and(DSL.field(fieldName).eq(dateTime));
             case NOT_EQUALS -> select = select.and(DSL.field(fieldName).ne(dateTime));
             default -> throw new UnsupportedOperationException("Unsupported operation for date condition: " + operation);
diff --git a/webserver/src/main/java/io/kestra/webserver/controllers/api/LogController.java b/webserver/src/main/java/io/kestra/webserver/controllers/api/LogController.java
index 751c04e84..4a7376ad8 100644
--- a/webserver/src/main/java/io/kestra/webserver/controllers/api/LogController.java
+++ b/webserver/src/main/java/io/kestra/webserver/controllers/api/LogController.java
@@ -95,9 +95,7 @@ public class LogController {
         TimeLineSearch timeLineSearch = TimeLineSearch.extractFrom(filters);
         validateTimeline(timeLineSearch.getStartDate(), timeLineSearch.getEndDate());
 
-        ZonedDateTime resolvedStartDate = RequestUtils.resolveAbsoluteDateTime(timeLineSearch.getStartDate(),
-            timeLineSearch.getTimeRange(),
-            now);
+        ZonedDateTime resolvedStartDate = timeLineSearch.getStartDate();
 
         // Update filters with the resolved startDate
         filters = QueryFilterUtils.updateFilters(filters, resolvedStartDate);
diff --git a/webserver/src/main/java/io/kestra/webserver/controllers/api/TaskRunController.java b/webserver/src/main/java/io/kestra/webserver/controllers/api/TaskRunController.java
index 97bb564a5..be847b8a3 100644
--- a/webserver/src/main/java/io/kestra/webserver/controllers/api/TaskRunController.java
+++ b/webserver/src/main/java/io/kestra/webserver/controllers/api/TaskRunController.java
@@ -88,9 +88,7 @@ public class TaskRunController {
         TimeLineSearch timeLineSearch = TimeLineSearch.extractFrom(filters);
         validateTimeline(timeLineSearch.getStartDate(), timeLineSearch.getEndDate());
 
-        ZonedDateTime resolvedStartDate = RequestUtils.resolveAbsoluteDateTime(timeLineSearch.getStartDate(),
-            timeLineSearch.getTimeRange(),
-            now);
+        ZonedDateTime resolvedStartDate = timeLineSearch.getStartDate();
 
         // Update filters with the resolved startDate
         filters = QueryFilterUtils.updateFilters(filters, resolvedStartDate);
diff --git a/webserver/src/main/java/io/kestra/webserver/utils/QueryFilterUtils.java b/webserver/src/main/java/io/kestra/webserver/utils/QueryFilterUtils.java
index e939b5937..ca32ce8f9 100644
--- a/webserver/src/main/java/io/kestra/webserver/utils/QueryFilterUtils.java
+++ b/webserver/src/main/java/io/kestra/webserver/utils/QueryFilterUtils.java
@@ -12,13 +12,20 @@ import java.util.List;
 public class QueryFilterUtils {
 
     public static List<QueryFilter> updateFilters(List<QueryFilter> filters, ZonedDateTime resolvedStartDate) {
+        boolean hasStartDateFilter = filters.stream().anyMatch(QueryFilterUtils::isStartDateFilter);
 
-        return filters.stream()
+        List<QueryFilter> updatedFilters = new java.util.ArrayList<>(filters.stream()
             .filter(filter -> !isTimeRangeFilter(filter)) // Remove TIME_RANGE filter
             .map(filter -> isStartDateFilter(filter)
                 ? createUpdatedStartDateFilter(filter, resolvedStartDate)
                 : filter)
-            .toList();
+            .toList());
+
+        if (!hasStartDateFilter && resolvedStartDate != null) {
+            updatedFilters.add(createUpdatedStartDateFilter(null, resolvedStartDate));
+        }
+
+        return updatedFilters;
     }
 
     private static boolean isStartDateFilter(QueryFilter filter) {
@@ -32,7 +39,7 @@ public class QueryFilterUtils {
     private static QueryFilter createUpdatedStartDateFilter(QueryFilter filter, ZonedDateTime resolvedStartDate) {
         return QueryFilter.builder()
             .field(QueryFilter.Field.START_DATE)
-            .operation(filter.operation())
+            .operation(filter != null ?filter.operation() : QueryFilter.Op.GREATER_THAN_OR_EQUAL_TO)
             .value(resolvedStartDate.toString())
             .build();
     }
diff --git a/webserver/src/main/java/io/kestra/webserver/utils/RequestUtils.java b/webserver/src/main/java/io/kestra/webserver/utils/RequestUtils.java
index 7ec534b37..9b00ce9cf 100644
--- a/webserver/src/main/java/io/kestra/webserver/utils/RequestUtils.java
+++ b/webserver/src/main/java/io/kestra/webserver/utils/RequestUtils.java
@@ -162,16 +162,4 @@ public class RequestUtils {
             .collect(Collectors.toList());
     }
 
-
-    public static ZonedDateTime resolveAbsoluteDateTime(ZonedDateTime absoluteDateTime, Duration timeRange, ZonedDateTime now) {
-        if (timeRange != null) {
-            if (absoluteDateTime != null) {
-                throw new IllegalArgumentException("Parameters 'startDate' and 'timeRange' are mutually exclusive");
-            }
-            return now.minus(timeRange.abs());
-        }
-
-        return absoluteDateTime;
-    }
-
 }


Test patch:
diff --git a/core/src/test/java/io/kestra/core/runners/pebble/functions/KvFunctionTest.java b/core/src/test/java/io/kestra/core/runners/pebble/functions/KvFunctionTest.java
index fcc6bd37a..bc4a6b81d 100644
--- a/core/src/test/java/io/kestra/core/runners/pebble/functions/KvFunctionTest.java
+++ b/core/src/test/java/io/kestra/core/runners/pebble/functions/KvFunctionTest.java
@@ -69,6 +69,45 @@ public class KvFunctionTest {
         assertThat(rendered, is("{\"field\":\"value\"}"));
     }
 
+    @Test
+    void shouldGetValueFromKVGivenExistingKeyWithInheritance() throws IllegalVariableEvaluationException, IOException {
+        // Given
+        KVStore kv = new InternalKVStore(null, "my.company", storageInterface);
+        kv.put("my-key", new KVValueAndMetadata(null, Map.of("field", "value")));
+
+        KVStore firstKv = new InternalKVStore(null, "my", storageInterface);
+        firstKv.put("my-key", new KVValueAndMetadata(null, Map.of("field", "firstValue")));
+
+        Map<String, Object> variables = Map.of(
+            "flow", Map.of(
+                "id", "kv",
+                "namespace", "my.company.team")
+        );
+
+        // When
+        String rendered = variableRenderer.render("{{ kv('my-key') }}", variables);
+
+        // Then
+        assertThat(rendered, is("{\"field\":\"value\"}"));
+    }
+
+    @Test
+    void shouldNotGetValueFromKVWithGivenNamespaceAndInheritance() throws IOException {
+        // Given
+        KVStore kv = new InternalKVStore(null, "kv", storageInterface);
+        kv.put("my-key", new KVValueAndMetadata(null, Map.of("field", "value")));
+
+        Map<String, Object> variables = Map.of(
+            "flow", Map.of(
+                "id", "kv",
+                "namespace", "my.company.team")
+        );
+
+        // When
+        Assertions.assertThrows(IllegalVariableEvaluationException.class, () ->
+            variableRenderer.render("{{ kv('my-key', namespace='kv.inherited') }}", variables));
+    }
+
     @Test
     void shouldGetValueFromKVGivenExistingAndNamespace() throws IllegalVariableEvaluationException, IOException {
         // Given
@@ -88,7 +127,6 @@ public class KvFunctionTest {
         assertThat(rendered, is("{\"field\":\"value\"}"));
     }
 
-
     @Test
     void shouldGetEmptyGivenNonExistingKeyAndErrorOnMissingFalse() throws IllegalVariableEvaluationException {
         // Given
diff --git a/core/src/test/java/io/kestra/plugin/core/kv/GetTest.java b/core/src/test/java/io/kestra/plugin/core/kv/GetTest.java
index 23c34a22e..01e29160e 100644
--- a/core/src/test/java/io/kestra/plugin/core/kv/GetTest.java
+++ b/core/src/test/java/io/kestra/plugin/core/kv/GetTest.java
@@ -60,6 +60,36 @@ class GetTest {
         assertThat(run.getValue(), is(value));
     }
 
+    @Test
+    void shouldGetGivenExistingKeyWithInheritance() throws Exception {
+        // Given
+        String namespaceId = "io.kestra." + IdUtils.create();
+        RunContext runContext = this.runContextFactory.of(Map.of(
+            "flow", Map.of("namespace", namespaceId),
+            "inputs", Map.of(
+                "key", TEST_KV_KEY
+            )
+        ));
+
+        var value = Map.of("date", Instant.now().truncatedTo(ChronoUnit.MILLIS), "int", 1, "string", "string");
+
+        Get get = Get.builder()
+            .id(Get.class.getSimpleName())
+            .type(Get.class.getName())
+            .key(new Property<>("{{ inputs.key }}"))
+            .build();
+
+
+        final KVStore kv = runContext.namespaceKv("io.kestra");
+
+        // When
+        kv.put(TEST_KV_KEY, new KVValueAndMetadata(null, value));
+
+        // Then
+        Get.Output run = get.run(runContext);
+        assertThat(run.getValue(), is(value));
+    }
+
     @Test
     void shouldGetGivenNonExistingKey() throws Exception {
         // Given
diff --git a/webserver/src/test/java/io/kestra/webserver/utils/RequestUtilsTest.java b/webserver/src/test/java/io/kestra/webserver/utils/RequestUtilsTest.java
index 4eade3661..9e0876f4b 100644
--- a/webserver/src/test/java/io/kestra/webserver/utils/RequestUtilsTest.java
+++ b/webserver/src/test/java/io/kestra/webserver/utils/RequestUtilsTest.java
@@ -85,21 +85,4 @@ class RequestUtilsTest {
         assertTrue(exception.getMessage().contains("Invalid FlowScope value"));
     }
 
-    @Test
-    void testResolveAbsoluteDateTimeWithTimeRange() {
-        ZonedDateTime now = ZonedDateTime.parse("2024-01-10T10:00:00Z");
-        Duration timeRange = Duration.ofDays(7);
-        ZonedDateTime resolved = RequestUtils.resolveAbsoluteDateTime(null, timeRange, now);
-
-        assertEquals(now.minus(timeRange), resolved);
-    }
-
-    @Test
-    void testResolveAbsoluteDateTimeWithAbsoluteDate() {
-        ZonedDateTime fixedDate = ZonedDateTime.parse("2024-01-01T10:00:00Z");
-        ZonedDateTime result = RequestUtils.resolveAbsoluteDateTime(fixedDate, null, ZonedDateTime.now());
-
-        assertEquals(fixedDate, result);
-    }
-
 }
\ No newline at end of file