Repository: DEFRA/trade-imports-processor
Pull request #32: CDMS-135: Discard old ClearanceRequests
Author: login: psmorgan
Created: 2025-04-15T13:44:47Z | Merged: 2025-04-15T14:32:26Z

We don't want to save ClearanceRequests that come through when there is a newer version stored within the Data Api.


Diff:
diff --git a/src/Processor/Consumers/CustomsDeclarationsConsumer.cs b/src/Processor/Consumers/CustomsDeclarationsConsumer.cs
index 4f1986d2..4d7887d3 100644
--- a/src/Processor/Consumers/CustomsDeclarationsConsumer.cs
+++ b/src/Processor/Consumers/CustomsDeclarationsConsumer.cs
@@ -23,7 +23,7 @@ public async Task OnHandle(JsonElement received, CancellationToken cancellationT
 
         var mrn = clearanceRequest.Header.EntryReference!;
 
-        var to = new DataApiCustomsDeclaration.ClearanceRequest
+        var newClearanceRequest = new DataApiCustomsDeclaration.ClearanceRequest
         {
             ExternalCorrelationId = clearanceRequest.ServiceHeader.CorrelationId,
             MessageSentAt = clearanceRequest.ServiceHeader.ServiceCallTimestamp,
@@ -73,16 +73,33 @@ public async Task OnHandle(JsonElement received, CancellationToken cancellationT
         if (existingCustomsDeclaration == null)
         {
             logger.LogInformation("Creating new customs declaration {Mrn}", mrn);
-            var newCustomsDeclaration = new DataApiCustomsDeclaration.CustomsDeclaration { ClearanceRequest = to };
+            var newCustomsDeclaration = new DataApiCustomsDeclaration.CustomsDeclaration
+            {
+                ClearanceRequest = newClearanceRequest,
+            };
 
             await api.PutCustomsDeclaration(mrn, newCustomsDeclaration, null, cancellationToken);
             return;
         }
 
+        if (
+            existingCustomsDeclaration.ClearanceRequest != null
+            && IsClearanceRequestOlderThan(newClearanceRequest, existingCustomsDeclaration.ClearanceRequest)
+        )
+        {
+            logger.LogInformation(
+                "Skipping {Mrn} because new clearance request {NewClearanceVersion} is older than existing {ExistingClearanceVersion}",
+                mrn,
+                newClearanceRequest.ExternalVersion,
+                existingCustomsDeclaration.ClearanceRequest.ExternalVersion
+            );
+            return;
+        }
+
         var updatedCustomsDeclaration = new DataApiCustomsDeclaration.CustomsDeclaration
         {
             ClearanceDecision = existingCustomsDeclaration.ClearanceDecision,
-            ClearanceRequest = to,
+            ClearanceRequest = newClearanceRequest,
             Finalisation = existingCustomsDeclaration.Finalisation,
         };
 
@@ -94,4 +111,12 @@ await api.PutCustomsDeclaration(
             cancellationToken
         );
     }
+
+    private static bool IsClearanceRequestOlderThan(
+        DataApiCustomsDeclaration.ClearanceRequest newClearanceRequest,
+        DataApiCustomsDeclaration.ClearanceRequest existingClearanceRequest
+    )
+    {
+        return newClearanceRequest.ExternalVersion < existingClearanceRequest.ExternalVersion;
+    }
 }
diff --git a/tests/Processor.Tests/Consumers/CustomsDeclarationsConsumerTests.cs b/tests/Processor.Tests/Consumers/CustomsDeclarationsConsumerTests.cs
index 468c9095..11a1467a 100644
--- a/tests/Processor.Tests/Consumers/CustomsDeclarationsConsumerTests.cs
+++ b/tests/Processor.Tests/Consumers/CustomsDeclarationsConsumerTests.cs
@@ -82,4 +82,38 @@ await _mockApi
                 _cancellationToken
             );
     }
+
+    [Fact]
+    public async Task OnHandle_WhenClearanceRequestReceived_ButExistingClearanceRequestIsNewer_ThenItIsSkipped()
+    {
+        var consumer = new CustomsDeclarationsConsumer(_mockLogger, _mockApi);
+
+        var mrn = GenerateMrn();
+        var clearanceRequest = ClearanceRequestFixture(mrn, 1).Create();
+        var existingClearanceRequest = DataApiClearanceRequestFixture().With(cr => cr.ExternalVersion, 2).Create();
+        const string expectedEtag = "12345";
+
+        var response = new CustomsDeclarationResponse(
+            mrn,
+            existingClearanceRequest,
+            null,
+            null,
+            DateTime.Now,
+            DateTime.Now,
+            expectedEtag
+        );
+
+        _mockApi.GetCustomsDeclaration(mrn, _cancellationToken).Returns(response);
+
+        await consumer.OnHandle(JsonSerializer.SerializeToElement(clearanceRequest), _cancellationToken);
+
+        await _mockApi
+            .DidNotReceiveWithAnyArgs()
+            .PutCustomsDeclaration(
+                Arg.Any<string>(),
+                Arg.Any<DataApiCustomsDeclaration.CustomsDeclaration>(),
+                Arg.Any<string>(),
+                Arg.Any<CancellationToken>()
+            );
+    }
 }
diff --git a/tests/TestFixtures/ClearanceRequestFixtures.cs b/tests/TestFixtures/ClearanceRequestFixtures.cs
index 0337e2af..4b85a556 100644
--- a/tests/TestFixtures/ClearanceRequestFixtures.cs
+++ b/tests/TestFixtures/ClearanceRequestFixtures.cs
@@ -13,20 +13,22 @@ private static Fixture GetFixture()
         return new Fixture();
     }
 
-    private static Header GenerateHeader(string? mrn = null)
+    private static Header GenerateHeader(int version, string? mrn = null)
     {
-        return GetFixture().Build<Header>().With(h => h.EntryReference, mrn ?? GenerateMrn()).Create();
+        return GetFixture()
+            .Build<Header>()
+            .With(h => h.EntryReference, mrn ?? GenerateMrn())
+            .With(h => h.EntryVersionNumber, version)
+            .Create();
     }
 
-    public static IPostprocessComposer<ClearanceRequest> ClearanceRequestFixture(string? mrn = null)
+    public static IPostprocessComposer<ClearanceRequest> ClearanceRequestFixture(string? mrn = null, int version = 2)
     {
-        return GetFixture().Build<ClearanceRequest>().With(c => c.Header, GenerateHeader(mrn));
+        return GetFixture().Build<ClearanceRequest>().With(c => c.Header, GenerateHeader(version, mrn));
     }
 
-    public static IPostprocessComposer<DataApiCustomsDeclaration.ClearanceRequest> DataApiClearanceRequestFixture(
-        string? mrn = null
-    )
+    public static IPostprocessComposer<DataApiCustomsDeclaration.ClearanceRequest> DataApiClearanceRequestFixture()
     {
-        return GetFixture().Build<DataApiCustomsDeclaration.ClearanceRequest>();
+        return GetFixture().Build<DataApiCustomsDeclaration.ClearanceRequest>().With(c => c.ExternalVersion, 1);
     }
 }


Reviews:
user: login: tal-nagra
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T14:15:22Z
user: login: psmorgan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T14:20:42Z
user: login: psmorgan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T14:20:55Z
user: login: psmorgan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T14:21:06Z
user: login: psmorgan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T14:23:18Z
user: login: t11omas
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T14:25:47Z
user: login: tal-nagra
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T14:27:17Z
user: login: tal-nagra
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-15T14:30:34Z

Review comments:
diff_hunk: @@ -23,7 +23,7 @@ public async Task OnHandle(JsonElement received, CancellationToken cancellationT
 
         var mrn = clearanceRequest.Header.EntryReference!;
 
-        var to = new DataApiCustomsDeclaration.ClearanceRequest
+        var newClearanceRequest = new DataApiCustomsDeclaration.ClearanceRequest
path: src/Processor/Consumers/CustomsDeclarationsConsumer.cs
user: login: tal-nagra
body: Consider moving this mapping to it's own method to make it a bit easier to see at a glance what is happening in `OnHandle`. This can be addressed in a future PR if you already have other refactoring etc in mind.

created_at: 2025-04-15T14:01:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -73,16 +73,31 @@ public async Task OnHandle(JsonElement received, CancellationToken cancellationT
         if (existingCustomsDeclaration == null)
         {
             logger.LogInformation("Creating new customs declaration {Mrn}", mrn);
-            var newCustomsDeclaration = new DataApiCustomsDeclaration.CustomsDeclaration { ClearanceRequest = to };
+            var newCustomsDeclaration = new DataApiCustomsDeclaration.CustomsDeclaration
+            {
+                ClearanceRequest = newClearanceRequest,
+            };
 
             await api.PutCustomsDeclaration(mrn, newCustomsDeclaration, null, cancellationToken);
             return;
         }
 
+        if (
+            existingCustomsDeclaration.ClearanceRequest != null
+            && NewClearanceRequestIsOlderThanExistingClearanceRequest(
+                newClearanceRequest,
+                existingCustomsDeclaration.ClearanceRequest
+            )
+        )
+        {
+            logger.LogInformation("Skipping {Mrn} because new clearance request is older than existing", mrn);
path: src/Processor/Consumers/CustomsDeclarationsConsumer.cs
user: login: tal-nagra
body: It would be worth adding the version numbers (old vs new) to the log entry here as they would be useful context if we ever have to investigate issues related to old clearance requests being received/skipped.
created_at: 2025-04-15T14:06:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -94,4 +109,12 @@ await api.PutCustomsDeclaration(
             cancellationToken
         );
     }
+
+    private static bool NewClearanceRequestIsOlderThanExistingClearanceRequest(
path: src/Processor/Consumers/CustomsDeclarationsConsumer.cs
user: login: tal-nagra
body: given the super clear parameter names of this method, this method could simply be called `IsClearanceRequestOlder` 
or slightly more verbose 'IsClearanceRequestOlderThanExisting`
created_at: 2025-04-15T14:08:24Z
author_association: CONTRIBUTOR
diff_hunk: @@ -82,4 +82,38 @@ await _mockApi
                 _cancellationToken
             );
     }
+
+    [Fact]
+    public async Task OnHandle_WhenClearanceRequestReceived_ButExistingClearanceRequestIsNewer_ThenItIsSkipped()
+    {
+        var consumer = new CustomsDeclarationsConsumer(_mockLogger, _mockApi);
+
+        var mrn = GenerateMrn();
+        var clearanceRequest = ClearanceRequestFixture(mrn, 1).Create();
+        var existingClearanceRequest = DataApiClearanceRequestFixture().With(cr => cr.ExternalVersion, 2).Create();
+        const string expectedEtag = "12345";
+
+        var response = new CustomsDeclarationResponse(
+            mrn,
+            existingClearanceRequest,
+            null,
+            null,
+            DateTime.Now,
+            DateTime.Now,
+            expectedEtag
+        );
+
+        _mockApi.GetCustomsDeclaration(mrn, _cancellationToken).Returns(response);
+
+        await consumer.OnHandle(JsonSerializer.SerializeToElement(clearanceRequest), _cancellationToken);
+
+        await _mockApi
+            .DidNotReceiveWithAnyArgs()
+            .PutCustomsDeclaration(
+                Arg.Any<string>(),
+                Arg.Any<DataApiCustomsDeclaration.CustomsDeclaration>(),
+                Arg.Any<string>(),
+                Arg.Any<CancellationToken>()
+            );
path: tests/Processor.Tests/Consumers/CustomsDeclarationsConsumerTests.cs
user: login: tal-nagra
body: Is there any value in adding an assertion for `_mockLogger` being invoked with a log message that starts with `"Skipping {mrn}"` ?
created_at: 2025-04-15T14:13:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -13,20 +13,22 @@ private static Fixture GetFixture()
         return new Fixture();
     }
 
-    private static Header GenerateHeader(string? mrn = null)
+    private static Header GenerateHeader(int version, string? mrn = null)
path: tests/TestFixtures/ClearanceRequestFixtures.cs
user: login: tal-nagra
body: Would it be helpful/appropriate to default the version to `1` ?
created_at: 2025-04-15T14:15:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -23,7 +23,7 @@ public async Task OnHandle(JsonElement received, CancellationToken cancellationT
 
         var mrn = clearanceRequest.Header.EntryReference!;
 
-        var to = new DataApiCustomsDeclaration.ClearanceRequest
+        var newClearanceRequest = new DataApiCustomsDeclaration.ClearanceRequest
path: src/Processor/Consumers/CustomsDeclarationsConsumer.cs
user: login: psmorgan
body: I'm kinda happy with how it is at the moment given the few processing rules that are in place for both consumers (is new > old).  At most yes the mappers could be moved in the future if necessary.
created_at: 2025-04-15T14:20:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -73,16 +73,31 @@ public async Task OnHandle(JsonElement received, CancellationToken cancellationT
         if (existingCustomsDeclaration == null)
         {
             logger.LogInformation("Creating new customs declaration {Mrn}", mrn);
-            var newCustomsDeclaration = new DataApiCustomsDeclaration.CustomsDeclaration { ClearanceRequest = to };
+            var newCustomsDeclaration = new DataApiCustomsDeclaration.CustomsDeclaration
+            {
+                ClearanceRequest = newClearanceRequest,
+            };
 
             await api.PutCustomsDeclaration(mrn, newCustomsDeclaration, null, cancellationToken);
             return;
         }
 
+        if (
+            existingCustomsDeclaration.ClearanceRequest != null
+            && NewClearanceRequestIsOlderThanExistingClearanceRequest(
+                newClearanceRequest,
+                existingCustomsDeclaration.ClearanceRequest
+            )
+        )
+        {
+            logger.LogInformation("Skipping {Mrn} because new clearance request is older than existing", mrn);
path: src/Processor/Consumers/CustomsDeclarationsConsumer.cs
user: login: psmorgan
body: 👍 
created_at: 2025-04-15T14:20:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -94,4 +109,12 @@ await api.PutCustomsDeclaration(
             cancellationToken
         );
     }
+
+    private static bool NewClearanceRequestIsOlderThanExistingClearanceRequest(
path: src/Processor/Consumers/CustomsDeclarationsConsumer.cs
user: login: psmorgan
body: 👍 
created_at: 2025-04-15T14:21:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -82,4 +82,38 @@ await _mockApi
                 _cancellationToken
             );
     }
+
+    [Fact]
+    public async Task OnHandle_WhenClearanceRequestReceived_ButExistingClearanceRequestIsNewer_ThenItIsSkipped()
+    {
+        var consumer = new CustomsDeclarationsConsumer(_mockLogger, _mockApi);
+
+        var mrn = GenerateMrn();
+        var clearanceRequest = ClearanceRequestFixture(mrn, 1).Create();
+        var existingClearanceRequest = DataApiClearanceRequestFixture().With(cr => cr.ExternalVersion, 2).Create();
+        const string expectedEtag = "12345";
+
+        var response = new CustomsDeclarationResponse(
+            mrn,
+            existingClearanceRequest,
+            null,
+            null,
+            DateTime.Now,
+            DateTime.Now,
+            expectedEtag
+        );
+
+        _mockApi.GetCustomsDeclaration(mrn, _cancellationToken).Returns(response);
+
+        await consumer.OnHandle(JsonSerializer.SerializeToElement(clearanceRequest), _cancellationToken);
+
+        await _mockApi
+            .DidNotReceiveWithAnyArgs()
+            .PutCustomsDeclaration(
+                Arg.Any<string>(),
+                Arg.Any<DataApiCustomsDeclaration.CustomsDeclaration>(),
+                Arg.Any<string>(),
+                Arg.Any<CancellationToken>()
+            );
path: tests/Processor.Tests/Consumers/CustomsDeclarationsConsumerTests.cs
user: login: psmorgan
body: I prefer to avoid testing log outputs, they should be informational and be not the required behaviour; the function would work the same with or without the log output.
created_at: 2025-04-15T14:23:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -82,4 +82,38 @@ await _mockApi
                 _cancellationToken
             );
     }
+
+    [Fact]
+    public async Task OnHandle_WhenClearanceRequestReceived_ButExistingClearanceRequestIsNewer_ThenItIsSkipped()
+    {
+        var consumer = new CustomsDeclarationsConsumer(_mockLogger, _mockApi);
+
+        var mrn = GenerateMrn();
+        var clearanceRequest = ClearanceRequestFixture(mrn, 1).Create();
+        var existingClearanceRequest = DataApiClearanceRequestFixture().With(cr => cr.ExternalVersion, 2).Create();
+        const string expectedEtag = "12345";
+
+        var response = new CustomsDeclarationResponse(
+            mrn,
+            existingClearanceRequest,
+            null,
+            null,
+            DateTime.Now,
+            DateTime.Now,
+            expectedEtag
+        );
+
+        _mockApi.GetCustomsDeclaration(mrn, _cancellationToken).Returns(response);
+
+        await consumer.OnHandle(JsonSerializer.SerializeToElement(clearanceRequest), _cancellationToken);
+
+        await _mockApi
+            .DidNotReceiveWithAnyArgs()
+            .PutCustomsDeclaration(
+                Arg.Any<string>(),
+                Arg.Any<DataApiCustomsDeclaration.CustomsDeclaration>(),
+                Arg.Any<string>(),
+                Arg.Any<CancellationToken>()
+            );
path: tests/Processor.Tests/Consumers/CustomsDeclarationsConsumerTests.cs
user: login: t11omas
body: I prefer to use NullLoggers (which is built into the framework), this removes the the incline to try and assert on log messages 
created_at: 2025-04-15T14:25:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -82,4 +82,38 @@ await _mockApi
                 _cancellationToken
             );
     }
+
+    [Fact]
+    public async Task OnHandle_WhenClearanceRequestReceived_ButExistingClearanceRequestIsNewer_ThenItIsSkipped()
+    {
+        var consumer = new CustomsDeclarationsConsumer(_mockLogger, _mockApi);
+
+        var mrn = GenerateMrn();
+        var clearanceRequest = ClearanceRequestFixture(mrn, 1).Create();
+        var existingClearanceRequest = DataApiClearanceRequestFixture().With(cr => cr.ExternalVersion, 2).Create();
+        const string expectedEtag = "12345";
+
+        var response = new CustomsDeclarationResponse(
+            mrn,
+            existingClearanceRequest,
+            null,
+            null,
+            DateTime.Now,
+            DateTime.Now,
+            expectedEtag
+        );
+
+        _mockApi.GetCustomsDeclaration(mrn, _cancellationToken).Returns(response);
+
+        await consumer.OnHandle(JsonSerializer.SerializeToElement(clearanceRequest), _cancellationToken);
+
+        await _mockApi
+            .DidNotReceiveWithAnyArgs()
+            .PutCustomsDeclaration(
+                Arg.Any<string>(),
+                Arg.Any<DataApiCustomsDeclaration.CustomsDeclaration>(),
+                Arg.Any<string>(),
+                Arg.Any<CancellationToken>()
+            );
path: tests/Processor.Tests/Consumers/CustomsDeclarationsConsumerTests.cs
user: login: tal-nagra
body: Fair enough. That's why I asked is there any value :slightly_smiling_face: 
created_at: 2025-04-15T14:27:17Z
author_association: CONTRIBUTOR

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-04-15T14:29:11Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=DEFRA_trade-imports-processor&pullRequest=32) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=DEFRA_trade-imports-processor&pullRequest=32&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=DEFRA_trade-imports-processor&pullRequest=32&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=DEFRA_trade-imports-processor&pullRequest=32&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [100.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=DEFRA_trade-imports-processor&pullRequest=32&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=DEFRA_trade-imports-processor&pullRequest=32&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=DEFRA_trade-imports-processor&pullRequest=32)


author_association: NONE