Repository: DEFRA/btms-backend
Pull request #215: CDMS-238 adds diagnostic/blob-replica endpoint to check connectivity
Author: login: craigedmunds
Created: 2025-03-18T11:21:35Z | Merged: 2025-03-18T13:59:40Z

To the second blob store & refactors some of the blob service code


Diff:
diff --git a/Btms.Backend/Endpoints/DiagnosticEndpoints.cs b/Btms.Backend/Endpoints/DiagnosticEndpoints.cs
index b37eff08..c475a53c 100644
--- a/Btms.Backend/Endpoints/DiagnosticEndpoints.cs
+++ b/Btms.Backend/Endpoints/DiagnosticEndpoints.cs
@@ -2,7 +2,9 @@
 using Btms.Backend.Asb;
 using Btms.Backend.Config;
 using Btms.BlobService;
+using Btms.Business;
 using Btms.Common.Extensions;
+using Btms.Replication;
 using HealthChecks.AzureServiceBus;
 using HealthChecks.AzureServiceBus.Configuration;
 using Microsoft.Extensions.Diagnostics.HealthChecks;
@@ -19,14 +21,24 @@ public static void UseDiagnosticEndpoints(this IEndpointRouteBuilder app, IOptio
         if (options.Value.EnableDiagnostics)
         {
             app.MapGet(BaseRoute + "/blob", GetBlobDiagnosticAsync).AllowAnonymous();
-            app.MapGet(BaseRoute + "/blob-replica", GetBlobDiagnosticAsync).AllowAnonymous();
+            app.MapGet(BaseRoute + "/blob-replica", GetReplicaBlobDiagnosticAsync).AllowAnonymous();
             app.MapGet(BaseRoute + "/asb", GetAzureServiceBusDiagnosticAsync).AllowAnonymous();
         }
     }
 
-    private static async Task<IResult> GetBlobDiagnosticAsync(IBlobService service)
+    private static async Task<IResult> GetBlobDiagnosticAsync(IBlobService service, IOptions<BusinessOptions> options)
     {
-        var result = await service.CheckBlobAsync(5, 1);
+        var result = await service.CheckBlobAsync(options.Value.DmpBlobRootFolder, 5, 1);
+        if (result.Success)
+        {
+            return Results.Ok(result);
+        }
+        return Results.Conflict(result);
+    }
+
+    private static async Task<IResult> GetReplicaBlobDiagnosticAsync(ReplicationTargetBlobService service)
+    {
+        var result = await service.CheckBlobAsync("", 5, 1);
         if (result.Success)
         {
             return Results.Ok(result);
diff --git a/Btms.Backend/appsettings.Development.json b/Btms.Backend/appsettings.Development.json
index b63b114e..e566c7a2 100644
--- a/Btms.Backend/appsettings.Development.json
+++ b/Btms.Backend/appsettings.Development.json
@@ -6,7 +6,8 @@
     "DatabaseName": "btms-backend"
   },
   "ApiOptions": {
-    "EnableManagement": true
+    "EnableManagement": true,
+    "EnableDiagnostics": true
   },
   "SensitiveDataOptions": {
     "Include": true
diff --git a/Btms.BlobService/BlobService.cs b/Btms.BlobService/BlobService.cs
index bc27fdec..9b2c238f 100644
--- a/Btms.BlobService/BlobService.cs
+++ b/Btms.BlobService/BlobService.cs
@@ -23,30 +23,38 @@ public abstract class BaseBlobService(
     IHttpClientFactory clientFactory)
     : AzureService(serviceProvider, logger, options.Value, clientFactory), IBlobService
 {
-    protected BlobContainerClient CreateBlobClient(int timeout = default, int retries = default)
+    protected BlobContainerClient CreateBlobClient(string uri, string container, int timeout = default, int retries = default)
     {
-        var blobServiceClient = blobServiceClientFactory.CreateBlobServiceClient(timeout, retries);
+        var blobServiceClient = blobServiceClientFactory.CreateBlobServiceClient(uri, timeout, retries);
 
-        var containerClient = blobServiceClient.GetBlobContainerClient(options.Value.DmpBlobContainer);
+        var containerClient = blobServiceClient.GetBlobContainerClient(container);
 
         return containerClient;
     }
-    public async Task<Status> CheckBlobAsync(int timeout = default, int retries = default)
+
+    private static string EnsurePrefixHasTrailingSlash(string prefix)
+    {
+        return !prefix.EndsWith('/') ? $"{prefix}/" : prefix;
+    }
+
+    public async Task<Status> CheckBlobAsync(string prefix = "RAW", int timeout = default, int retries = default)
     {
-        return await CheckBlobAsync(options.Value.DmpBlobUri, timeout, retries);
+        return await CheckBlobAsync(options.Value.DmpBlobUri, options.Value.DmpBlobContainer, prefix, timeout, retries);
     }
 
-    public async Task<Status> CheckBlobAsync(string uri, int timeout = default, int retries = default)
+    public async Task<Status> CheckBlobAsync(string uri, string container, string prefix = "RAW", int timeout = default, int retries = default)
     {
-        Logger.LogInformation("Connecting to blob storage {Uri} : {BlobContainer}. timeout={Timeout}, retries={Retries}",
-            uri, options.Value.DmpBlobContainer, timeout, retries);
+        prefix = EnsurePrefixHasTrailingSlash(prefix);
+
+        Logger.LogInformation("Connecting to blob storage {Uri} : {BlobContainer}/{Prefix}. timeout={Timeout}, retries={Retries}",
+            uri, container, prefix, timeout, retries);
 
         try
         {
-            var containerClient = CreateBlobClient(timeout, retries);
+            var containerClient = CreateBlobClient(uri, container, timeout, retries);
 
-            Logger.LogInformation("Getting blob folders...");
-            var folders = containerClient.GetBlobsByHierarchyAsync(prefix: "RAW/", delimiter: "/");
+            Logger.LogInformation("Getting blob folders from {Prefix}...", prefix);
+            var folders = containerClient.GetBlobsByHierarchyAsync(prefix: prefix, delimiter: "/");
 
             var itemCount = 0;
             await foreach (var blobItem in folders)
@@ -58,7 +66,7 @@ public async Task<Status> CheckBlobAsync(string uri, int timeout = default, int
             return new Status
             {
                 Success = true,
-                Description = $"Connected. {itemCount} blob folders found in RAW"
+                Description = $"Connected. {itemCount} blob folders found in {prefix}"
             };
         }
         catch (Exception ex)
@@ -74,10 +82,12 @@ public async Task<Status> CheckBlobAsync(string uri, int timeout = default, int
             "Ignored this is IAsyncEnumerable and doesn't support linq filtering out the box")]
     public async IAsyncEnumerable<IBlobItem> GetResourcesAsync(string prefix, [EnumeratorCancellation] CancellationToken cancellationToken)
     {
+        prefix = EnsurePrefixHasTrailingSlash(prefix);
+
         Logger.LogDebug("Connecting to blob storage {BlobUri} : {BlobContainer} : {Path}", options.Value.DmpBlobUri,
             options.Value.DmpBlobContainer, prefix);
 
-        var containerClient = CreateBlobClient();
+        var containerClient = CreateBlobClient(options.Value.DmpBlobUri, options.Value.DmpBlobContainer);
 
         var itemCount = 0;
 
@@ -99,7 +109,7 @@ public async IAsyncEnumerable<IBlobItem> GetResourcesAsync(string prefix, [Enume
 
     public async Task<string> GetResource(IBlobItem item, CancellationToken cancellationToken)
     {
-        var client = CreateBlobClient(options.Value.Timeout, options.Value.Retries);
+        var client = CreateBlobClient(options.Value.DmpBlobUri, options.Value.DmpBlobContainer, options.Value.Timeout, options.Value.Retries);
         var blobClient = client.GetBlobClient(item.Name);
 
         var content = await blobClient.DownloadContentAsync(cancellationToken);
diff --git a/Btms.BlobService/BlobServiceClientFactory.cs b/Btms.BlobService/BlobServiceClientFactory.cs
index 25d215d6..9d71a3c5 100644
--- a/Btms.BlobService/BlobServiceClientFactory.cs
+++ b/Btms.BlobService/BlobServiceClientFactory.cs
@@ -13,6 +13,10 @@ public class BlobServiceClientFactory(
     : AzureService(serviceProvider, logger, options.Value, clientFactory), IBlobServiceClientFactory
 {
     public BlobServiceClient CreateBlobServiceClient(int timeout = default, int retries = default)
+    {
+        return CreateBlobServiceClient(options.Value.DmpBlobUri);
+    }
+    public BlobServiceClient CreateBlobServiceClient(string uri, int timeout = default, int retries = default)
     {
         // Allow timeout and retry to be overridden, e.g. from healthchecker
         timeout = timeout > 0 ? timeout : options.Value.Timeout;
@@ -30,9 +34,8 @@ public BlobServiceClient CreateBlobServiceClient(int timeout = default, int retr
             Diagnostics = { IsLoggingContentEnabled = true, IsLoggingEnabled = true }
         };
 
-
         return new BlobServiceClient(
-            new Uri(options.Value.DmpBlobUri),
+            new Uri(uri),
             Credentials,
             bcOptions);
     }
diff --git a/Btms.BlobService/CachingBlobService.cs b/Btms.BlobService/CachingBlobService.cs
index ee83876c..1dfad186 100644
--- a/Btms.BlobService/CachingBlobService.cs
+++ b/Btms.BlobService/CachingBlobService.cs
@@ -11,14 +11,14 @@ public class CachingBlobService(
     IOptions<BlobServiceOptions> options
 ) : IBlobService
 {
-    public Task<Status> CheckBlobAsync(int timeout = default, int retries = default)
+    public Task<Status> CheckBlobAsync(string prefix = "RAW", int timeout = default, int retries = default)
     {
-        return blobService.CheckBlobAsync(timeout, retries);
+        return blobService.CheckBlobAsync(prefix, timeout, retries);
     }
 
-    public Task<Status> CheckBlobAsync(string uri, int timeout = default, int retries = default)
+    public Task<Status> CheckBlobAsync(string uri, string container, string prefix = "RAW", int timeout = default, int retries = default)
     {
-        return blobService.CheckBlobAsync(uri, timeout, retries);
+        return blobService.CheckBlobAsync(uri, container, prefix, timeout, retries);
     }
 
     public async IAsyncEnumerable<IBlobItem> GetResourcesAsync(string prefix, [EnumeratorCancellation] CancellationToken cancellationToken)
@@ -70,7 +70,7 @@ public async Task<string> GetResource(IBlobItem item, CancellationToken cancella
 
         var filePath = $"{options.Value.CachePath}/{item.Name}";
         logger.LogInformation("GetResource {FilePath}", filePath);
-        return File.ReadAllText(filePath);
+        return await File.ReadAllTextAsync(filePath, cancellationToken);
     }
 
     public async Task<bool> CreateBlobsAsync(IBlobItem[] items)
diff --git a/Btms.BlobService/IBlobService.cs b/Btms.BlobService/IBlobService.cs
index 15d01abc..a3cd5311 100644
--- a/Btms.BlobService/IBlobService.cs
+++ b/Btms.BlobService/IBlobService.cs
@@ -2,8 +2,8 @@ namespace Btms.BlobService;
 
 public interface IBlobService
 {
-    public Task<Status> CheckBlobAsync(int timeout = default, int retries = default);
-    public Task<Status> CheckBlobAsync(string uri, int timeout = default, int retries = default);
+    public Task<Status> CheckBlobAsync(string prefix = "RAW", int timeout = default, int retries = default);
+    public Task<Status> CheckBlobAsync(string uri, string container, string prefix = "RAW", int timeout = default, int retries = default);
     public IAsyncEnumerable<IBlobItem> GetResourcesAsync(string prefix, CancellationToken cancellationToken);
     public Task<string> GetResource(IBlobItem item, CancellationToken cancellationToken);
     public Task<bool> CreateBlobsAsync(IBlobItem[] items);
diff --git a/Btms.BlobService/IBlobServiceClientFactory.cs b/Btms.BlobService/IBlobServiceClientFactory.cs
index 90acb4c4..bcdaac17 100644
--- a/Btms.BlobService/IBlobServiceClientFactory.cs
+++ b/Btms.BlobService/IBlobServiceClientFactory.cs
@@ -5,4 +5,5 @@ namespace Btms.BlobService;
 public interface IBlobServiceClientFactory
 {
     BlobServiceClient CreateBlobServiceClient(int timeout = default, int retries = default);
+    BlobServiceClient CreateBlobServiceClient(string uri, int timeout = default, int retries = default);
 }
\ No newline at end of file
diff --git a/Btms.Replication/Commands/ReplicateCommand.cs b/Btms.Replication/Commands/ReplicateCommand.cs
index 037bb556..960edf6d 100644
--- a/Btms.Replication/Commands/ReplicateCommand.cs
+++ b/Btms.Replication/Commands/ReplicateCommand.cs
@@ -23,8 +23,8 @@ public class ReplicateCommand() : IRequest, ISyncJob
     public SyncPeriod SyncPeriod { get; set; }
 
     public Guid JobId { get; } = Guid.NewGuid();
-    public string Timespan { get; } = null!;
-    public string Resource { get; } = null!;
+    public string Timespan => SyncPeriod.ToString();
+    public string Resource { get; } = "Replicate";
 
     public string? RootFolder { get; set; }
 
@@ -45,10 +45,11 @@ public async Task Handle(ReplicateCommand request, CancellationToken cancellatio
 
                 await DownloadCommand.BlobFolders.ForEachAsync(async f =>
                 {
-                    logger.LogInformation("Replicate {Path}", f.path);
+                    logger.LogInformation("Replicate {Path} for syncPeriod {Period}", f.path, request.SyncPeriod);
 
                     var blobs = blobService.GetBlobItems($"{blobFolder}/{f.path}", request.SyncPeriod, cancellationToken);
 
+
                     await Parallel.ForEachAsync(blobs, cancellationToken, async (item, ct) =>
                     {
                         logger.LogWarning("Processing file {Name}", item.Name);
diff --git a/Btms.Replication/ReplicationTargetBlobService.cs b/Btms.Replication/ReplicationTargetBlobService.cs
index e53635a8..d6b27290 100644
--- a/Btms.Replication/ReplicationTargetBlobService.cs
+++ b/Btms.Replication/ReplicationTargetBlobService.cs
@@ -13,7 +13,7 @@ public class ReplicationTargetBlobService(IServiceProvider serviceProvider,
 {
     public async Task WriteResource(string path, string content, CancellationToken cancellationToken)
     {
-        var client = CreateBlobClient(options.Value.Timeout, options.Value.Retries);
+        var client = CreateBlobClient(options.Value.DmpBlobUri, options.Value.DmpBlobContainer, options.Value.Timeout, options.Value.Retries);
         var blobClient = client.GetBlobClient(path);
 
         var result = await blobClient.UploadAsync(BinaryData.FromString(content), cancellationToken);


Reviews:
user: login: t11omas
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T13:09:37Z
user: login: t11omas
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T13:37:43Z
user: login: craigedmunds
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T13:55:58Z

Review comments:
diff_hunk: @@ -19,14 +21,24 @@ public static void UseDiagnosticEndpoints(this IEndpointRouteBuilder app, IOptio
         if (options.Value.EnableDiagnostics)
         {
             app.MapGet(BaseRoute + "/blob", GetBlobDiagnosticAsync).AllowAnonymous();
-            app.MapGet(BaseRoute + "/blob-replica", GetBlobDiagnosticAsync).AllowAnonymous();
+            app.MapGet(BaseRoute + "/blob-replica", GetReplicaBlobDiagnosticAsync).AllowAnonymous();
path: Btms.Backend/Endpoints/DiagnosticEndpoints.cs
user: login: t11omas
body: Should this also be added to the health endpoint?
created_at: 2025-03-18T13:09:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -19,14 +21,24 @@ public static void UseDiagnosticEndpoints(this IEndpointRouteBuilder app, IOptio
         if (options.Value.EnableDiagnostics)
         {
             app.MapGet(BaseRoute + "/blob", GetBlobDiagnosticAsync).AllowAnonymous();
-            app.MapGet(BaseRoute + "/blob-replica", GetBlobDiagnosticAsync).AllowAnonymous();
+            app.MapGet(BaseRoute + "/blob-replica", GetReplicaBlobDiagnosticAsync).AllowAnonymous();
path: Btms.Backend/Endpoints/DiagnosticEndpoints.cs
user: login: craigedmunds
body: I didn't on account its not essential to operation and moving soon.
created_at: 2025-03-18T13:55:58Z
author_association: CONTRIBUTOR

PR comments:
user: login: sonarqubecloud[bot]
created_at: 2025-03-18T12:02:08Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=DEFRA_btms-backend&pullRequest=215) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 New issues](https://sonarcloud.io/project/issues?id=DEFRA_btms-backend&pullRequest=215&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=DEFRA_btms-backend&pullRequest=215&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=DEFRA_btms-backend&pullRequest=215&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=DEFRA_btms-backend&pullRequest=215&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=DEFRA_btms-backend&pullRequest=215&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=DEFRA_btms-backend&pullRequest=215)


author_association: NONE
user: login: github-actions[bot]
created_at: 2025-03-18T12:07:54Z
body: ![Code Coverage](https://img.shields.io/badge/Code%20Coverage-78%25-yellow?style=flat)

Package | Line Rate | Branch Rate | Health
-------- | --------- | ----------- | ------
Btms.Model | 77% | 54% | ➖
Btms.Types.Ipaffs.Mapping | 77% | 59% | ➖
Btms.SyncJob | 61% | 38% | ➖
Btms.Common | 72% | 52% | ➖
Btms.BlobService | 47% | 32% | ❌
Btms.Metrics | 81% | 0% | ✔
Btms.Analytics | 74% | 74% | ➖
Btms.Analytics.Export | 6% | 0% | ❌
Btms.Business | 87% | 80% | ✔
Btms.Backend.Data | 61% | 48% | ➖
Btms.Types.Alvs.Mapping.V1.Tests | 0% | 0% | ❌
Btms.Validation | 82% | 93% | ✔
Btms.SensitiveData | 87% | 95% | ✔
Btms.Types.Gvms | 78% | 100% | ➖
Btms.Azure | 74% | 100% | ➖
Btms.Emf | 11% | 0% | ❌
Btms.Replication | 84% | 25% | ✔
Btms.Consumers | 78% | 70% | ➖
Btms.Model.Validation | 100% | 54% | ✔
Btms.Types.Alvs.Mapping | 80% | 0% | ✔
Btms.Types.Gvms.Mapping | 68% | 38% | ➖
Btms.Types.Alvs | 87% | 100% | ✔
Btms.Types.Ipaffs | 95% | 62% | ✔
Btms.Backend | 51% | 34% | ❌
Btms.Types.Alvs.Validation | 94% | 56% | ✔
TestGenerator.IntegrationTesting.Backend | 81% | 70% | ✔
TestDataGenerator | 87% | 81% | ✔
**Summary** | **78%** (12622 / 16261) | **62%** (1633 / 2624) | ➖

<!-- Sticky Pull Request Comment -->
author_association: NONE
user: login: t11omas
created_at: 2025-03-18T13:10:52Z
body: Few warnings on BTMS replication too.  should errors as warnings be turned on
author_association: CONTRIBUTOR