Repository: tmfg/digitraffic-tis-vaco
Pull request #434: TIS-980/logic fixes to complete entry
Author: login: solita-areta
Created: 2025-03-21T12:12:52Z | Merged: 2025-03-24T09:47:52Z



Diff:
diff --git a/src/main/java/fi/digitraffic/tis/vaco/delegator/DelegationJobQueueSqsListener.java b/src/main/java/fi/digitraffic/tis/vaco/delegator/DelegationJobQueueSqsListener.java
index dad8a510..e77dac11 100644
--- a/src/main/java/fi/digitraffic/tis/vaco/delegator/DelegationJobQueueSqsListener.java
+++ b/src/main/java/fi/digitraffic/tis/vaco/delegator/DelegationJobQueueSqsListener.java
@@ -105,7 +105,7 @@ protected void runTask(ImmutableDelegationJobMessage message) {
                 entryService.updateStatus(entry);
                 notificationsService.notifyEntryComplete(entry);
             } else {
-                // some kind of limbo/crash
+                logger.info("Entry {} still has tasks running, will wait for completion", entry.publicId());
             }
         }
     }
diff --git a/src/main/java/fi/digitraffic/tis/vaco/rules/RuleResultsListener.java b/src/main/java/fi/digitraffic/tis/vaco/rules/RuleResultsListener.java
index 4cb443ac..609642b1 100644
--- a/src/main/java/fi/digitraffic/tis/vaco/rules/RuleResultsListener.java
+++ b/src/main/java/fi/digitraffic/tis/vaco/rules/RuleResultsListener.java
@@ -5,9 +5,13 @@
 import com.google.common.annotations.VisibleForTesting;
 import fi.digitraffic.tis.aws.sqs.SqsListener;
 import fi.digitraffic.tis.utilities.model.ProcessingState;
+import fi.digitraffic.tis.vaco.db.repositories.FindingRepository;
 import fi.digitraffic.tis.vaco.entries.EntryService;
 import fi.digitraffic.tis.vaco.entries.model.Status;
 import fi.digitraffic.tis.vaco.findings.FindingService;
+import fi.digitraffic.tis.vaco.findings.model.Finding;
+import fi.digitraffic.tis.vaco.findings.model.FindingSeverity;
+import fi.digitraffic.tis.vaco.findings.model.ImmutableFinding;
 import fi.digitraffic.tis.vaco.messaging.MessagingService;
 import fi.digitraffic.tis.vaco.messaging.model.ImmutableDelegationJobMessage;
 import fi.digitraffic.tis.vaco.messaging.model.ImmutableRetryStatistics;
@@ -29,6 +33,7 @@
 import fi.digitraffic.tis.vaco.rules.results.ResultProcessor;
 import fi.digitraffic.tis.vaco.rules.results.SimpleResultProcessor;
 import fi.digitraffic.tis.vaco.summary.SummaryService;
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.scheduling.annotation.Scheduled;
@@ -57,6 +62,7 @@ public class RuleResultsListener extends SqsListener {
     private final SummaryService summaryService;
     private final GtfsToNetexResultProcessor gtfsToNetexResultProcessor;
     private final NetexToGtfsRuleResultProcessor netexToGtfsRuleResultProcessor;
+    private final FindingRepository findingRepository;
 
     public RuleResultsListener(MessagingService messagingService,
                                FindingService findingService,
@@ -71,7 +77,8 @@ public RuleResultsListener(MessagingService messagingService,
                                InternalRuleResultProcessor internalRuleResultProcessor,
                                SummaryService summaryService,
                                GtfsToNetexResultProcessor gtfsToNetexResultProcessor,
-                               NetexToGtfsRuleResultProcessor netexToGtfsRuleResultProcessor) {
+                               NetexToGtfsRuleResultProcessor netexToGtfsRuleResultProcessor,
+                               FindingRepository findingRepository) {
         super(messagingService, objectMapper);
         this.findingService = Objects.requireNonNull(findingService);
         this.queueHandlerService = Objects.requireNonNull(queueHandlerService);
@@ -85,6 +92,7 @@ public RuleResultsListener(MessagingService messagingService,
         this.summaryService = Objects.requireNonNull(summaryService);
         this.gtfsToNetexResultProcessor = Objects.requireNonNull(gtfsToNetexResultProcessor);
         this.netexToGtfsRuleResultProcessor = Objects.requireNonNull(netexToGtfsRuleResultProcessor);
+        this.findingRepository = Objects.requireNonNull(findingRepository);
     }
 
     @Scheduled(initialDelayString = "${vaco.scheduling.findings.poll-rate}", fixedRateString = "${vaco.scheduling.findings.poll-rate}")
@@ -115,26 +123,31 @@ protected CompletableFuture<Boolean> handleDeadLetter(JsonNode jsonNode) {
         return CompletableFuture.supplyAsync(() -> {
             logger.info("Dead letter Queue Message {}", jsonNode);
 
+            Optional<Finding> finding = createDeadLetterQueueFinding(jsonNode);
+            finding.ifPresent(findingRepository::create);
+
             Optional<String> taskPublicId = Optional.ofNullable(jsonNode)
                 .filter(node -> node.has("task"))
                 .map(node -> node.get("task").get("publicId"))
                 .map(JsonNode::asText);
 
+            Optional<Entry> entry = getEntryPublicId(jsonNode)
+                .flatMap(entryService::findEntry);
+
             return taskPublicId
                 .flatMap(taskService::findTask)
                 .map(task -> {
                     taskService.markStatus(task, Status.FAILED);
+                    entry.ifPresent(e -> taskService.trackTask(e, task, ProcessingState.COMPLETE));
                     return true;
                 }).orElse(false);
         }).whenComplete((deadLetterProcessingSuccess, maybeEx) -> {
             if (maybeEx != null) {
-                logger.warn("Handling deal letter queue message failed due to unhandled exception", maybeEx);
+                logger.warn("Handling dead letter queue message failed due to unhandled exception", maybeEx);
             }
             // resubmit to processing queue to continue general logic
-            Optional<String> entryPublicId = Optional.ofNullable(jsonNode)
-                .filter(node -> node.has("entry"))
-                .map(node -> node.get("entry").get("publicId"))
-                .map(JsonNode::asText);
+            Optional<String> entryPublicId = getEntryPublicId(jsonNode);
+
             entryPublicId.flatMap(entryService::findEntry)
                 .ifPresent(entry ->
                     messagingService.submitProcessingJob(ImmutableDelegationJobMessage.builder()
@@ -144,6 +157,43 @@ protected CompletableFuture<Boolean> handleDeadLetter(JsonNode jsonNode) {
         });
     }
 
+    private Optional<Finding> createDeadLetterQueueFinding(JsonNode jsonNode) {
+
+        Optional<String> entryId = getEntryPublicId(jsonNode);
+
+        Optional<String> taskName = Optional.ofNullable(jsonNode)
+            .filter(node -> node.has("task"))
+            .map(node -> node.get("task").get("name"))
+            .map(JsonNode::asText);
+
+        return entryId.flatMap(e -> {
+            Optional<Long> taskId = Optional.ofNullable(jsonNode)
+                .filter(node -> node.has("task"))
+                .map(node -> node.get("task").get("id"))
+                .map(JsonNode::asLong);
+
+            if (taskName.isPresent()) {
+                return taskId.map(t -> ImmutableFinding.builder()
+                    .publicId(e)
+                    .source(taskName.get())
+                    .taskId(t)
+                    .message("Entry ended up in dead letter queue ")
+                    .severity(FindingSeverity.ERROR)
+                    .build());
+            } else {
+                return Optional.empty();
+            }
+        });
+    }
+
+    @NotNull
+    private static Optional<String> getEntryPublicId(JsonNode jsonNode) {
+        return Optional.ofNullable(jsonNode)
+            .filter(node -> node.has("entry"))
+            .map(node -> node.get("entry").get("publicId"))
+            .map(JsonNode::asText);
+    }
+
     protected CompletableFuture<Boolean> handleResult(ResultMessage resultMessage) {
         return CompletableFuture.supplyAsync(() -> {
             logger.debug("Got ResultMessage {}", resultMessage);
diff --git a/src/test/java/fi/digitraffic/tis/vaco/rules/RuleResultsListenerTests.java b/src/test/java/fi/digitraffic/tis/vaco/rules/RuleResultsListenerTests.java
index e1289e75..9c332382 100644
--- a/src/test/java/fi/digitraffic/tis/vaco/rules/RuleResultsListenerTests.java
+++ b/src/test/java/fi/digitraffic/tis/vaco/rules/RuleResultsListenerTests.java
@@ -1,15 +1,23 @@
 package fi.digitraffic.tis.vaco.rules;
 
+import com.aventrix.jnanoid.jnanoid.NanoIdUtils;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.guava.GuavaModule;
 import fi.digitraffic.tis.Constants;
+import fi.digitraffic.tis.utilities.model.ProcessingState;
 import fi.digitraffic.tis.vaco.TestObjects;
 import fi.digitraffic.tis.vaco.configuration.VacoProperties;
+import fi.digitraffic.tis.vaco.db.model.FindingRecord;
+import fi.digitraffic.tis.vaco.db.model.ImmutableFindingRecord;
+import fi.digitraffic.tis.vaco.db.repositories.FindingRepository;
 import fi.digitraffic.tis.vaco.entries.EntryService;
 import fi.digitraffic.tis.vaco.entries.model.Status;
 import fi.digitraffic.tis.vaco.findings.FindingService;
+import fi.digitraffic.tis.vaco.findings.model.Finding;
+import fi.digitraffic.tis.vaco.findings.model.FindingSeverity;
+import fi.digitraffic.tis.vaco.findings.model.ImmutableFinding;
 import fi.digitraffic.tis.vaco.messaging.MessagingService;
 import fi.digitraffic.tis.vaco.messaging.model.DelegationJobMessage;
 import fi.digitraffic.tis.vaco.messaging.model.QueueNames;
@@ -43,6 +51,7 @@
 
 import java.util.Map;
 import java.util.Optional;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -76,6 +85,7 @@ class RuleResultsListenerTests {
     @Mock private NetexToGtfsRuleResultProcessor netexToGtfsRuleResultProcessor;
     @Mock private SummaryService summaryService;
     @Captor private ArgumentCaptor<DelegationJobMessage> submittedProcessingJob;
+    @Mock private FindingRepository findingRepository;
 
     @BeforeEach
     void setUp() {
@@ -96,7 +106,8 @@ void setUp() {
             internalRuleResultProcessor,
             summaryService,
             gtfsToNetexResultProcessor,
-            netexToGtfsRuleResultProcessor);
+            netexToGtfsRuleResultProcessor,
+            findingRepository);
     }
 
     @AfterEach
@@ -111,7 +122,10 @@ void tearDown() {
             gbfsResultProcessor,
             gtfsCanonicalProcessor,
             simpleResultProcessor,
-            internalRuleResultProcessor);
+            internalRuleResultProcessor,
+            gtfsToNetexResultProcessor,
+            netexToGtfsRuleResultProcessor,
+            findingRepository);
     }
 
     @Test
@@ -140,18 +154,45 @@ void deadLetterQueueTest() throws JsonProcessingException {
         String jsonString = """
                 {
                   "entry": { "publicId":"abc1"},
-                  "task": { "id":2, "publicId":"abc2"}
+                  "task": { "id":2, "publicId":"abc2", "name": "gtfs.canonical" }
                 }
             """;
 
         JsonNode message = objectMapper.readValue(jsonString, JsonNode.class);
-        Task task = ImmutableTask.of("dlqTest", 100).withPublicId("abc2");
+        Task task = ImmutableTask.of("gtfs.canonical", 100).withPublicId("abc2");
         given(taskService.markStatus(task, Status.FAILED)).willReturn(task);
         given(taskService.findTask(task.publicId())).willReturn(Optional.of(task));
-        Entry entry = ImmutableEntry.of("abc1", "dlq test entry", "gtfs", "http://example.fintraffic", Constants.FINTRAFFIC_BUSINESS_ID, false);
+
+        ImmutableEntry.Builder entryBuilder = TestObjects.anEntry("gtfs");
+        Entry entry = entryBuilder.addTasks(task)
+            .publicId("abc1")
+            .name("dlq test entry")
+            .format("gtfs")
+            .url("http://example.fintraffic")
+            .businessId(Constants.FINTRAFFIC_BUSINESS_ID)
+            .sendNotifications(false)
+            .build();
+
         given(entryService.findEntry(entry.publicId())).willReturn(Optional.of(entry));
         givenResultProcessingResultsInNewProcessingJobSubmission();
 
+        Finding finding = ImmutableFinding.builder()
+            .publicId(entry.publicId())
+            .source(task.name())
+            .taskId(2L)
+            .message("Entry ended up in dead letter queue ")
+            .severity(FindingSeverity.ERROR)
+            .build();
+
+        FindingRecord findingRecord = ImmutableFindingRecord.builder()
+            .id(500L)
+            .publicId(NanoIdUtils.randomNanoId())
+            .taskId(500L)
+            .source(task.name())
+            .build();
+
+        given(findingRepository.create(finding)).willReturn(findingRecord);
+        given(taskService.trackTask(entry, task, ProcessingState.COMPLETE)).willReturn(task);
         assertThat(ruleResultsListener.handleDeadLetter(message).join(), equalTo(true));
     }
 


Reviews:
user: login: esuomi
body: Some code style changes and such still needed.
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-21T12:29:03Z
user: login: esuomi
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-24T08:57:38Z

Review comments:
diff_hunk: @@ -105,7 +105,7 @@ protected void runTask(ImmutableDelegationJobMessage message) {
                 entryService.updateStatus(entry);
                 notificationsService.notifyEntryComplete(entry);
             } else {
-                // some kind of limbo/crash
+                logger.info("Entry {} still has task running, will wait for completion", entry.publicId());
path: src/main/java/fi/digitraffic/tis/vaco/delegator/DelegationJobQueueSqsListener.java
user: login: esuomi
body: ```suggestion
                logger.info("Entry {} still has tasks running, will wait for completion", entry.publicId());
```
created_at: 2025-03-21T12:23:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -115,26 +123,34 @@ protected CompletableFuture<Boolean> handleDeadLetter(JsonNode jsonNode) {
         return CompletableFuture.supplyAsync(() -> {
             logger.info("Dead letter Queue Message {}", jsonNode);
 
+            Optional<Finding> finding = createDeadLetterQueueFinding(jsonNode);
+            finding.ifPresent(findingRepository::create);
+
             Optional<String> taskPublicId = Optional.ofNullable(jsonNode)
                 .filter(node -> node.has("task"))
                 .map(node -> node.get("task").get("publicId"))
                 .map(JsonNode::asText);
 
+            Optional<String> entryPublicId = getEntryPublicId(jsonNode);
+
+            Optional<Entry> entry = entryPublicId
+                .flatMap(entryService::findEntry);
path: src/main/java/fi/digitraffic/tis/vaco/rules/RuleResultsListener.java
user: login: esuomi
body: ```suggestion
            Optional<Entry> entry = getEntryPublicId(jsonNode)
                .flatMap(entryService::findEntry);
```
created_at: 2025-03-21T12:24:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -115,26 +123,34 @@ protected CompletableFuture<Boolean> handleDeadLetter(JsonNode jsonNode) {
         return CompletableFuture.supplyAsync(() -> {
             logger.info("Dead letter Queue Message {}", jsonNode);
 
+            Optional<Finding> finding = createDeadLetterQueueFinding(jsonNode);
+            finding.ifPresent(findingRepository::create);
+
             Optional<String> taskPublicId = Optional.ofNullable(jsonNode)
                 .filter(node -> node.has("task"))
                 .map(node -> node.get("task").get("publicId"))
                 .map(JsonNode::asText);
 
+            Optional<String> entryPublicId = getEntryPublicId(jsonNode);
+
+            Optional<Entry> entry = entryPublicId
+                .flatMap(entryService::findEntry);
+
             return taskPublicId
                 .flatMap(taskService::findTask)
                 .map(task -> {
                     taskService.markStatus(task, Status.FAILED);
+                    entry.ifPresent(e ->
+                        taskService.trackTask(e, task, ProcessingState.COMPLETE));
path: src/main/java/fi/digitraffic/tis/vaco/rules/RuleResultsListener.java
user: login: esuomi
body: ```suggestion
                    entry.ifPresent(e -> taskService.trackTask(e, task, ProcessingState.COMPLETE));
```
created_at: 2025-03-21T12:24:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -115,26 +123,34 @@ protected CompletableFuture<Boolean> handleDeadLetter(JsonNode jsonNode) {
         return CompletableFuture.supplyAsync(() -> {
             logger.info("Dead letter Queue Message {}", jsonNode);
 
+            Optional<Finding> finding = createDeadLetterQueueFinding(jsonNode);
+            finding.ifPresent(findingRepository::create);
+
             Optional<String> taskPublicId = Optional.ofNullable(jsonNode)
                 .filter(node -> node.has("task"))
                 .map(node -> node.get("task").get("publicId"))
                 .map(JsonNode::asText);
 
+            Optional<String> entryPublicId = getEntryPublicId(jsonNode);
+
+            Optional<Entry> entry = entryPublicId
+                .flatMap(entryService::findEntry);
+
             return taskPublicId
                 .flatMap(taskService::findTask)
                 .map(task -> {
                     taskService.markStatus(task, Status.FAILED);
+                    entry.ifPresent(e ->
+                        taskService.trackTask(e, task, ProcessingState.COMPLETE));
                     return true;
                 }).orElse(false);
         }).whenComplete((deadLetterProcessingSuccess, maybeEx) -> {
             if (maybeEx != null) {
-                logger.warn("Handling deal letter queue message failed due to unhandled exception", maybeEx);
+                logger.warn("Handling dead letter queue message failed due to unhandled exception", maybeEx);
path: src/main/java/fi/digitraffic/tis/vaco/rules/RuleResultsListener.java
user: login: esuomi
body: There should probably be `return;` after this, as `deadLetterProcessingSuccess` is always false at this point anyway...hmm, not sure actually, please check if you can.
created_at: 2025-03-21T12:26:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -140,18 +151,45 @@ void deadLetterQueueTest() throws JsonProcessingException {
         String jsonString = """
                 {
                   "entry": { "publicId":"abc1"},
-                  "task": { "id":2, "publicId":"abc2"}
+                  "task": { "id":2, "publicId":"abc2", "name": "gtfs.canonical" }
                 }
             """;
 
         JsonNode message = objectMapper.readValue(jsonString, JsonNode.class);
-        Task task = ImmutableTask.of("dlqTest", 100).withPublicId("abc2");
+        Task task = ImmutableTask.of("gtfs.canonical", 100).withPublicId("abc2");
         given(taskService.markStatus(task, Status.FAILED)).willReturn(task);
         given(taskService.findTask(task.publicId())).willReturn(Optional.of(task));
-        Entry entry = ImmutableEntry.of("abc1", "dlq test entry", "gtfs", "http://example.fintraffic", Constants.FINTRAFFIC_BUSINESS_ID, false);
+
+        ImmutableEntry.Builder entryBuilder = TestObjects.anEntry("gtfs");
+        Entry entry = entryBuilder.addTasks(task)
+            .publicId("abc1")
+            .name("dlq test entry")
+            .format("gtfs")
+            .url("http://example.fintraffic")
+            .businessId(Constants.FINTRAFFIC_BUSINESS_ID)
+            .sendNotifications(false)
+            .build();
+
         given(entryService.findEntry(entry.publicId())).willReturn(Optional.of(entry));
         givenResultProcessingResultsInNewProcessingJobSubmission();
 
+        Finding finding = ImmutableFinding.builder()
+            .publicId(entry.publicId())
+            .source(task.name())
+            .taskId(2L)
+            .message("Entry ended up in dead letter queue ")
+            .severity(FindingSeverity.ERROR)
+            .build();
+
+        FindingRecord findingRecord = ImmutableFindingRecord.builder()
+            .id(500L)
+            .publicId(String.valueOf(UUID.randomUUID()))
path: src/test/java/fi/digitraffic/tis/vaco/rules/RuleResultsListenerTests.java
user: login: esuomi
body: ```suggestion
            .publicId(NanoIdUtils.randomNanoId()))
```

Remember to add the import as well :)
created_at: 2025-03-21T12:28:02Z
author_association: CONTRIBUTOR

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-03-24T07:10:25Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=tmfg_digitraffic-tis-vaco&pullRequest=434) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [2 New issues](https://sonarcloud.io/project/issues?id=tmfg_digitraffic-tis-vaco&pullRequest=434&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=tmfg_digitraffic-tis-vaco&pullRequest=434&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=tmfg_digitraffic-tis-vaco&pullRequest=434&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [97.1% Coverage on New Code](https://sonarcloud.io/component_measures?id=tmfg_digitraffic-tis-vaco&pullRequest=434&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=tmfg_digitraffic-tis-vaco&pullRequest=434&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=tmfg_digitraffic-tis-vaco&pullRequest=434)


author_association: NONE