Repository: sillsdev/serval
Pull request #670: Clearml monitor service tests
Author: login: mudiagaobrikisil
Created: 2025-04-09T14:03:45Z | Merged: 2025-04-28T16:59:39Z

A collection of unit tests for the Clear ML Monitor Service

<!-- Reviewable:start -->
- - -
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/sillsdev/serval/670)
<!-- Reviewable:end -->



Diff:
diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs
index 320a0d7cf..9e3e94a4b 100644
--- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs
+++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs
@@ -17,9 +17,9 @@ ILogger<ClearMLMonitorService> logger
     ),
         IClearMLQueueService
 {
-    private static readonly string SummaryMetric = CreateMD5("Summary");
-    private static readonly string TrainCorpusSizeVariant = CreateMD5("train_corpus_size");
-    private static readonly string ConfidenceVariant = CreateMD5("confidence");
+    internal static readonly string SummaryMetric = CreateMD5("Summary");
+    internal static readonly string TrainCorpusSizeVariant = CreateMD5("train_corpus_size");
+    internal static readonly string ConfidenceVariant = CreateMD5("confidence");
 
     private readonly IClearMLService _clearMLService = clearMLService;
     private readonly ISharedFileService _sharedFileService = sharedFileService;
@@ -43,7 +43,7 @@ protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken
         await MonitorClearMLTasksPerDomain(scope, cancellationToken);
     }
 
-    private async Task MonitorClearMLTasksPerDomain(IServiceScope scope, CancellationToken cancellationToken)
+    internal async Task MonitorClearMLTasksPerDomain(IServiceScope scope, CancellationToken cancellationToken)
     {
         try
         {
diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLMonitorServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLMonitorServiceTests.cs
new file mode 100644
index 000000000..4833de186
--- /dev/null
+++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLMonitorServiceTests.cs
@@ -0,0 +1,483 @@
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Serval.Machine.Shared.Services;
+
+[TestFixture]
+public class ClearMLMonitorServiceTests
+{
+    // Test constants
+    private const string DefaultEngineId = "engine-123";
+    private const string DefaultBuildId = "build-456";
+    private const string DefaultJobId = "job-789";
+    private const string DefaultProjectId = "proj1";
+
+    // Service under test
+    private ClearMLMonitorService _service;
+
+    // Dependencies
+    private IClearMLService _clearMLService;
+    private ISharedFileService _sharedFileService;
+    private ILogger<ClearMLMonitorService> _logger;
+    private IServiceProvider _serviceProvider;
+    private IOptionsMonitor<BuildJobOptions> _buildJobOptions;
+    private IOptionsMonitor<ClearMLOptions> _clearMLOptions;
+    private IDataAccessContext _dataAccessContext;
+    private IPlatformService _platformService;
+    private IBuildJobService<TranslationEngine> _translationBuildJobService;
+    private IBuildJobService<WordAlignmentEngine> _wordAlignmentBuildJobService;
+
+    [SetUp]
+    public void Setup()
+    {
+        // Create mocks for all dependencies
+        SetupMocks();
+
+        // Configure common mock behaviors
+        ConfigureMockBehaviors();
+
+        // Create service provider
+        _serviceProvider = CreateServiceProvider();
+
+        // Create service under test
+        _service = new ClearMLMonitorService(
+            _serviceProvider,
+            _clearMLService,
+            _sharedFileService,
+            _clearMLOptions,
+            _buildJobOptions,
+            _logger
+        );
+    }
+
+    private void SetupMocks()
+    {
+        _clearMLService = Substitute.For<IClearMLService>();
+        _sharedFileService = Substitute.For<ISharedFileService>();
+        _logger = Substitute.For<ILogger<ClearMLMonitorService>>();
+        _dataAccessContext = Substitute.For<IDataAccessContext>();
+        _platformService = Substitute.For<IPlatformService>();
+        _translationBuildJobService = Substitute.For<IBuildJobService<TranslationEngine>>();
+        _wordAlignmentBuildJobService = Substitute.For<IBuildJobService<WordAlignmentEngine>>();
+
+        _buildJobOptions = Substitute.For<IOptionsMonitor<BuildJobOptions>>();
+        _clearMLOptions = Substitute.For<IOptionsMonitor<ClearMLOptions>>();
+    }
+
+    private void ConfigureMockBehaviors()
+    {
+        _buildJobOptions.CurrentValue.Returns(
+            new BuildJobOptions { ClearML = [new ClearMLBuildQueue { EngineType = EngineType.Nmt, Queue = "default" }] }
+        );
+
+        _clearMLOptions.CurrentValue.Returns(
+            new ClearMLOptions { BuildPollingTimeout = TimeSpan.FromSeconds(10), BuildPollingEnabled = true }
+        );
+    }
+
+    private IServiceProvider CreateServiceProvider()
+    {
+        ServiceCollection serviceCollection = new ServiceCollection();
+
+        // Register services
+        serviceCollection.AddScoped(_ => _translationBuildJobService);
+        serviceCollection.AddScoped(_ => _wordAlignmentBuildJobService);
+        serviceCollection.AddScoped(_ => _dataAccessContext);
+        serviceCollection.AddScoped(_ => _platformService);
+
+        // Register keyed services
+        serviceCollection.AddKeyedScoped<IPlatformService>(EngineGroup.Translation, (_, _) => _platformService);
+        serviceCollection.AddKeyedScoped<IPlatformService>(EngineGroup.WordAlignment, (_, _) => _platformService);
+
+        return serviceCollection.BuildServiceProvider();
+    }
+
+    // Helper method to create test engines
+    private static TranslationEngine CreateTestEngine(
+        string engineId = DefaultEngineId,
+        string buildId = DefaultBuildId,
+        string jobId = DefaultJobId,
+        BuildJobState jobState = BuildJobState.Pending,
+        BuildStage stage = BuildStage.Train
+    )
+    {
+        return new TranslationEngine
+        {
+            EngineId = engineId,
+            Type = EngineType.Nmt,
+            CurrentBuild = new Build
+            {
+                BuildId = buildId,
+                JobId = jobId,
+                JobState = jobState,
+                BuildJobRunner = BuildJobRunnerType.ClearML,
+                Stage = stage
+            },
+            SourceLanguage = "en",
+            TargetLanguage = "fr",
+            IsModelPersisted = true,
+        };
+    }
+
+    // Helper method to create ClearML tasks
+    private static ClearMLTask CreateClearMLTask(
+        string id,
+        string name,
+        ClearMLTaskStatus status,
+        DateTime created,
+        Dictionary<string, string>? runtime = null,
+        Dictionary<string, IReadOnlyDictionary<string, ClearMLMetricsEvent>>? lastMetrics = null,
+        int lastIteration = 0,
+        string? statusMessage = null,
+        string? statusReason = null
+    )
+    {
+        return new ClearMLTask
+        {
+            Id = id,
+            Name = name,
+            Status = status,
+            Created = created,
+            Runtime = runtime ?? new Dictionary<string, string>(),
+            LastMetrics = lastMetrics ?? new Dictionary<string, IReadOnlyDictionary<string, ClearMLMetricsEvent>>(),
+            LastIteration = lastIteration,
+            StatusMessage = statusMessage,
+            StatusReason = statusReason,
+            Project = new ClearMLProject { Id = DefaultProjectId }
+        };
+    }
+
+    [Test]
+    public async Task MonitorClearMLTasksPerDomain_QueuedStatus_UpdatesQueuePosition()
+    {
+        TranslationEngine engine = CreateTestEngine();
+        SetupBuildingEngines(engine);
+
+        ClearMLTask task = CreateClearMLTask(
+            id: engine!.CurrentBuild!.JobId,
+            name: engine.CurrentBuild.BuildId,
+            status: ClearMLTaskStatus.Queued,
+            created: DateTime.UtcNow
+        );
+
+        SetupClearMLTasks(task);
+
+        using IServiceScope scope = _serviceProvider.CreateScope();
+        await _service.MonitorClearMLTasksPerDomain(scope, CancellationToken.None);
+
+        int queueSize = _service.GetQueueSize(EngineType.Nmt);
+        Assert.That(queueSize, Is.EqualTo(1));
+
+        await VerifyStatusUpdate(
+            buildId: engine.CurrentBuild.BuildId,
+            step: 0,
+            percentCompleted: 0.0,
+            queuePosition: null // null means we don't care about the value
+        );
+    }
+
+    [Test]
+    public async Task MonitorClearMLTasksPerDomain_InProgress_UpdatesProgressCorrectly()
+    {
+        TranslationEngine engine = CreateTestEngine(jobState: BuildJobState.Active);
+        SetupBuildingEngines(engine);
+
+        Dictionary<string, string> runtimeInfo = new Dictionary<string, string>
+        {
+            { "progress", "50" },
+            { "message", "Training epoch 5/10" }
+        };
+
+        ClearMLTask task = CreateClearMLTask(
+            id: engine!.CurrentBuild!.JobId,
+            name: engine.CurrentBuild.BuildId,
+            status: ClearMLTaskStatus.InProgress,
+            created: DateTime.UtcNow,
+            runtime: runtimeInfo,
+            lastIteration: 5
+        );
+
+        SetupClearMLTasks(task);
+
+        using IServiceScope scope = _serviceProvider.CreateScope();
+        await _service.MonitorClearMLTasksPerDomain(scope, CancellationToken.None);
+
+        // Assert
+        await VerifyStatusUpdate(
+            buildId: engine.CurrentBuild.BuildId,
+            step: 5,
+            percentCompleted: 0.5,
+            queuePosition: 0,
+            message: "Training epoch 5/10"
+        );
+    }
+
+    [Test]
+    public async Task MonitorClearMLTasksPerDomain_CompletedStatus_ProperlyHandlesCompletion()
+    {
+        const int ExpectedCorpusSize = 1000;
+        const double ExpectedConfidence = 0.95;
+
+        TranslationEngine engine = CreateTestEngine(jobState: BuildJobState.Active, stage: BuildStage.Train);
+        SetupBuildingEngines(engine);
+
+        Dictionary<string, IReadOnlyDictionary<string, ClearMLMetricsEvent>> lastMetrics = new Dictionary<
+            string,
+            IReadOnlyDictionary<string, ClearMLMetricsEvent>
+        >
+        {
+            [ClearMLMonitorService.SummaryMetric] = new Dictionary<string, ClearMLMetricsEvent>
+            {
+                [ClearMLMonitorService.TrainCorpusSizeVariant] = new ClearMLMetricsEvent { Value = ExpectedCorpusSize },
+                [ClearMLMonitorService.ConfidenceVariant] = new ClearMLMetricsEvent { Value = ExpectedConfidence }
+            }
+        };
+
+        Dictionary<string, string> runtime = new Dictionary<string, string> { ["message"] = "Training complete" };
+
+        ClearMLTask task = CreateClearMLTask(
+            id: engine!.CurrentBuild!.JobId,
+            name: engine.CurrentBuild.BuildId,
+            status: ClearMLTaskStatus.Completed,
+            created: DateTime.UtcNow,
+            runtime: runtime,
+            lastMetrics: lastMetrics,
+            lastIteration: 100
+        );
+
+        SetupClearMLTasks(task);
+
+        _translationBuildJobService
+            .StartBuildJobAsync(
+                BuildJobRunnerType.Hangfire,
+                engine.Type,
+                engine.EngineId,
+                engine.CurrentBuild.BuildId,
+                BuildStage.Postprocess,
+                Arg.Is<(int, double)>(x => x.Item1 == ExpectedCorpusSize && x.Item2 == ExpectedConfidence),
+                engine.CurrentBuild.Options,
+                Arg.Any<CancellationToken>()
+            )
+            .Returns(true);
+
+        using IServiceScope scope = _serviceProvider.CreateScope();
+        await _service.MonitorClearMLTasksPerDomain(scope, CancellationToken.None);
+
+        // Assert - Verify status update
+        await VerifyStatusUpdate(
+            buildId: engine.CurrentBuild.BuildId,
+            step: 100,
+            percentCompleted: 1.0,
+            queuePosition: 0,
+            message: "Training complete"
+        );
+
+        // Assert - Verify postprocessing job started
+        await _translationBuildJobService
+            .Received(1)
+            .StartBuildJobAsync(
+                BuildJobRunnerType.Hangfire,
+                engine.Type,
+                engine.EngineId,
+                engine.CurrentBuild.BuildId,
+                BuildStage.Postprocess,
+                Arg.Is<(int, double)>(x => x.Item1 == ExpectedCorpusSize && x.Item2 == ExpectedConfidence),
+                engine.CurrentBuild.Options,
+                Arg.Any<CancellationToken>()
+            );
+    }
+
+    [Test]
+    public async Task MonitorClearMLTasksPerDomain_StoppedStatus_CancelsBuildAndCleansUp()
+    {
+        TranslationEngine engine = CreateTestEngine(jobState: BuildJobState.Active, stage: BuildStage.Train);
+
+        // Create real data access context
+        MemoryDataAccessContext dataAccessContext = new MemoryDataAccessContext();
+        IServiceProvider serviceProvider = CreateServiceProviderWithRealDataContext(dataAccessContext);
+
+        // Create service with real data context
+        _service = new ClearMLMonitorService(
+            serviceProvider,
+            _clearMLService,
+            _sharedFileService,
+            _clearMLOptions,
+            _buildJobOptions,
+            _logger
+        );
+
+        // Setup building engines
+        SetupBuildingEngines(engine);
+
+        // Setup stopped task
+        ClearMLTask task = CreateClearMLTask(
+            id: engine!.CurrentBuild!.JobId,
+            name: engine.CurrentBuild.BuildId,
+            status: ClearMLTaskStatus.Stopped,
+            created: DateTime.UtcNow,
+            statusMessage: "Manually stopped",
+            statusReason: "User requested stop"
+        );
+
+        SetupClearMLTasks(task);
+
+        using IServiceScope scope = serviceProvider.CreateScope();
+        await _service.MonitorClearMLTasksPerDomain(scope, CancellationToken.None);
+
+        // Assert
+        await VerifyCancellationFlow(engine);
+    }
+
+    [Test]
+    public async Task MonitorClearMLTasksPerDomain_FailedStatus_ProperlyHandlesFailure()
+    {
+        const string ErrorMessage = "Training failed due to GPU memory error";
+        const string ErrorReason = "RuntimeError";
+
+        TranslationEngine engine = CreateTestEngine(jobState: BuildJobState.Active, stage: BuildStage.Train);
+
+        // Use real implementations
+        MemoryDataAccessContext dataAccessContext = new MemoryDataAccessContext();
+        _sharedFileService = new SharedFileService(Substitute.For<ILoggerFactory>());
+
+        // Create dummy files
+        await _sharedFileService.OpenWriteAsync($"builds/{engine!.CurrentBuild!.BuildId}/model.bin");
+        await _sharedFileService.OpenWriteAsync($"builds/{engine.CurrentBuild.BuildId}/logs.txt");
+
+        // Setup service with real implementations
+        IServiceProvider serviceProvider = CreateServiceProviderWithRealDataContext(dataAccessContext);
+        _service = new ClearMLMonitorService(
+            serviceProvider,
+            _clearMLService,
+            _sharedFileService,
+            _clearMLOptions,
+            _buildJobOptions,
+            _logger
+        );
+
+        // Setup building engines
+        SetupBuildingEngines(engine);
+
+        // Setup failed task
+        ClearMLTask task = CreateClearMLTask(
+            id: engine.CurrentBuild.JobId,
+            name: engine.CurrentBuild.BuildId,
+            status: ClearMLTaskStatus.Failed,
+            created: DateTime.UtcNow,
+            statusMessage: ErrorMessage,
+            statusReason: ErrorReason
+        );
+
+        SetupClearMLTasks(task);
+
+        using IServiceScope scope = serviceProvider.CreateScope();
+        await _service.MonitorClearMLTasksPerDomain(scope, CancellationToken.None);
+
+        // Assert
+        await VerifyFailureHandling(engine, ErrorReason, ErrorMessage);
+
+        // Verify files were actually deleted
+        var files = await _sharedFileService.ListFilesAsync($"builds/{engine.CurrentBuild.BuildId}/");
+        Assert.That(files, Is.Empty, "Build files should have been deleted");
+    }
+
+    #region Helper Methods
+
+    private void SetupBuildingEngines(TranslationEngine engine)
+    {
+        _translationBuildJobService
+            .GetBuildingEnginesAsync(BuildJobRunnerType.ClearML, Arg.Any<CancellationToken>())
+            .Returns(new[] { engine });
+    }
+
+    private void SetupClearMLTasks(ClearMLTask task)
+    {
+        _clearMLService
+            .GetTasksByIdAsync(Arg.Any<IEnumerable<string>>(), Arg.Any<CancellationToken>())
+            .Returns(new[] { task });
+    }
+
+    private IServiceProvider CreateServiceProviderWithRealDataContext(IDataAccessContext dataAccessContext)
+    {
+        ServiceCollection serviceCollection = new ServiceCollection();
+        serviceCollection.AddScoped<IDataAccessContext>(_ => dataAccessContext);
+        serviceCollection.AddScoped(_ => _translationBuildJobService);
+        serviceCollection.AddScoped(_ => _wordAlignmentBuildJobService);
+        serviceCollection.AddScoped(_ => _platformService);
+        serviceCollection.AddKeyedScoped<IPlatformService>(EngineGroup.Translation, (_, _) => _platformService);
+        serviceCollection.AddKeyedScoped<IPlatformService>(EngineGroup.WordAlignment, (_, _) => _platformService);
+
+        return serviceCollection.BuildServiceProvider();
+    }
+
+    private async Task VerifyStatusUpdate(
+        string buildId,
+        int step,
+        double percentCompleted,
+        int? queuePosition = null,
+        string? message = null
+    )
+    {
+        if (queuePosition.HasValue)
+        {
+            await _platformService
+                .Received(1)
+                .UpdateBuildStatusAsync(
+                    buildId,
+                    Arg.Is<ProgressStatus>(s =>
+                        s.Step == step
+                        && s.PercentCompleted == percentCompleted
+                        && (message == null || s.Message == message)
+                    ),
+                    queuePosition.Value,
+                    Arg.Any<CancellationToken>()
+                );
+        }
+        else
+        {
+            await _platformService
+                .Received(1)
+                .UpdateBuildStatusAsync(
+                    buildId,
+                    Arg.Is<ProgressStatus>(s =>
+                        s.Step == step
+                        && s.PercentCompleted == percentCompleted
+                        && (message == null || s.Message == message)
+                    ),
+                    Arg.Any<int>(),
+                    Arg.Any<CancellationToken>()
+                );
+        }
+    }
+
+    private async Task VerifyCancellationFlow(TranslationEngine engine)
+    {
+        await _platformService
+            .Received(1)
+            .BuildCanceledAsync(engine!.CurrentBuild!.BuildId, Arg.Any<CancellationToken>());
+
+        await _translationBuildJobService
+            .Received(1)
+            .BuildJobFinishedAsync(engine.EngineId, engine.CurrentBuild.BuildId, false, Arg.Any<CancellationToken>());
+
+        await _sharedFileService
+            .Received(1)
+            .DeleteAsync($"builds/{engine.CurrentBuild.BuildId}/", Arg.Any<CancellationToken>());
+    }
+
+    private async Task VerifyFailureHandling(TranslationEngine engine, string errorReason, string errorMessage)
+    {
+        await _platformService
+            .Received(1)
+            .BuildFaultedAsync(
+                engine!.CurrentBuild!.BuildId,
+                Arg.Is<string>(msg => msg.Contains(errorReason) && msg.Contains(errorMessage)),
+                Arg.Any<CancellationToken>()
+            );
+
+        await _translationBuildJobService
+            .Received(1)
+            .BuildJobFinishedAsync(engine.EngineId, engine.CurrentBuild.BuildId, false, Arg.Any<CancellationToken>());
+    }
+
+    #endregion
+}


Reviews:
user: login: copilot-pull-request-reviewer[bot]
body: 

Copilot reviewed 3 out of 4 changed files in this pull request and generated 1 comment.

<details>
<summary>Files not reviewed (1)</summary>

* **src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj**: Language not supported
</details>



state: COMMENTED
author_association: NONE
submitted_at: 2025-04-10T18:02:22Z
user: login: johnml1135
body: 





<!-- Sent from Reviewable.io -->

state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-10T18:10:16Z
user: login: mudiagaobrikisil
body: 

> *[Reviewable](https://reviewable.io/reviews/sillsdev/serval/670)* status: 0 of 4 files reviewed, 4 unresolved discussions (waiting on @copilot-pull-request-reviewer[bot] and @johnml1135)

___
*[`src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs` line 2 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVscyAEc7I_kA-UJHm:-ONWBuxo3HAKX9iTnmkr:btmjfhp) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs#L2)):*
<details><summary><i>Previously, johnml1135 (John Lambert) wrote…</i></summary><blockquote>

Why is this needed?
</blockquote></details>

Ensures the methods marked as internal are visible within the namespace
___
*[`src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs` line 251 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVsjMe7-o_99gHIvPB:-ONWCSPC6gC5UvGRQX4u:bb71kmx) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs#L251)):*
<details><summary><i>Previously, johnml1135 (John Lambert) wrote…</i></summary><blockquote>

I am a little confused - I would believe that it would be better to call the top level function and just setup the inputs and outputs so that all lower level functions are called.
</blockquote></details>

Okay
___
*[`src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj` line 57 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVrpBgAbNUlFFplkHo:-ONWBc1w36rdPnf2iyHu:bf9j6ox) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj#L57)):*
<details><summary><i>Previously, johnml1135 (John Lambert) wrote…</i></summary><blockquote>

Why is this change needed?  I believe we are wanting to stay at .NET 8 for LTS until .NET 10 comes out - https://learn.microsoft.com/en-us/dotnet/core/whats-new/dotnet-10/overview
</blockquote></details>

Oh Sorry. I forgot to undo that. Done now


<!-- Sent from Reviewable.io -->

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T19:38:39Z
user: login: mudiagaobrikisil
body: 

> *[Reviewable](https://reviewable.io/reviews/sillsdev/serval/670)* status: 0 of 4 files reviewed, 4 unresolved discussions (waiting on @copilot-pull-request-reviewer[bot] and @johnml1135)



<!-- Sent from Reviewable.io -->

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T19:39:42Z
user: login: mudiagaobrikisil
body: 

> *[Reviewable](https://reviewable.io/reviews/sillsdev/serval/670)* status: 0 of 4 files reviewed, 4 unresolved discussions (waiting on @copilot-pull-request-reviewer[bot] and @johnml1135)

___
*[`src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs` line 251 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVsjMe7-o_99gHIvPB:-ONWDqbkAe9ppHORAa_g:brhcjju) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs#L251)):*
<details><summary><i>Previously, mudiagaobrikisil wrote…</i></summary><blockquote>

Okay
</blockquote></details>

But none of these functions call that top level function. Only the DoWorkAsync does that


<!-- Sent from Reviewable.io -->

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T19:45:40Z
user: login: mudiagaobrikisil
body: 

> *[Reviewable](https://reviewable.io/reviews/sillsdev/serval/670)* status: 0 of 4 files reviewed, 4 unresolved discussions (waiting on @copilot-pull-request-reviewer[bot] and @johnml1135)

___
*[`src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs` line 251 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVsjMe7-o_99gHIvPB:-ONWGoL46sdJxfcAmXHI:b-d8raaq) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs#L251)):*
<details><summary><i>Previously, mudiagaobrikisil wrote…</i></summary><blockquote>

But none of these functions call that top level function. Only the DoWorkAsync does that
</blockquote></details>

Thanks for the explanation. Understood


<!-- Sent from Reviewable.io -->

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-10T19:57:39Z
user: login: ddaspit
body: 

> Reviewed 1 of 4 files at r1, 3 of 3 files at r3, all commit messages.
> *[Reviewable](https://reviewable.io/reviews/sillsdev/serval/670)* status: all files reviewed, 5 unresolved discussions (waiting on @copilot-pull-request-reviewer[bot] and @johnml1135)

___
*[`src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs` line 2 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVscyAEc7I_kA-UJHm:-OOPOK_z4hKX3MhALw5o:b-l85smn) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs#L2)):*
<details><summary><i>Previously, mudiagaobrikisil wrote…</i></summary><blockquote>

Ensures the methods marked as internal are visible within the namespace
</blockquote></details>

This shouldn't be necessary. You can remove it.
___
*[`src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLMonitorServiceTests.cs` line 152 at r3](https://reviewable.io/reviews/sillsdev/serval/670#-OOPPVP4DYYSg3SVsWut:-OOPPVP4DYYSg3SVsWuu:b-a7ygqb) ([raw file](https://github.com/sillsdev/serval/blob/707dc954d2f7cd511ad89e672a70d62e39cf281f/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLMonitorServiceTests.cs#L152)):*
> ```Smalltalk
>     public async Task MonitorClearMLTasksPerDomain_QueuedStatus_UpdatesQueuePosition()
>     {
>         var engine = CreateTestEngine();
> ```

Please only use `var` when the type is explicit elsewhere in the statement.


<!-- Sent from Reviewable.io -->

state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-21T22:19:02Z
user: login: mudiagaobrikisil
body: 

> *[Reviewable](https://reviewable.io/reviews/sillsdev/serval/670)* status: 2 of 4 files reviewed, 5 unresolved discussions (waiting on @copilot-pull-request-reviewer[bot], @ddaspit, and @johnml1135)

___
*[`src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLMonitorServiceTests.cs` line 152 at r3](https://reviewable.io/reviews/sillsdev/serval/670#-OOPPVP4DYYSg3SVsWut:-OOSJO7z5uKEA6kQjvkZ:bb6un1j) ([raw file](https://github.com/sillsdev/serval/blob/707dc954d2f7cd511ad89e672a70d62e39cf281f/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLMonitorServiceTests.cs#L152)):*
<details><summary><i>Previously, ddaspit (Damien Daspit) wrote…</i></summary><blockquote>

Please only use `var` when the type is explicit elsewhere in the statement.
</blockquote></details>

Done
___
*[`src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs` line 2 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVscyAEc7I_kA-UJHm:-OOSJIByE0ZssoLwJC8E:b-4rmtg9) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs#L2)):*
<details><summary><i>Previously, ddaspit (Damien Daspit) wrote…</i></summary><blockquote>

This shouldn't be necessary. You can remove it.
</blockquote></details>

I have removed it


<!-- Sent from Reviewable.io -->

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-22T11:46:08Z
user: login: ddaspit
body: 

> Reviewed 2 of 2 files at r4, all commit messages.
> Dismissed @johnml1135 from 2 discussions.
> *[Reviewable](https://reviewable.io/reviews/sillsdev/serval/670)* status: all files reviewed, 1 unresolved discussion (waiting on @johnml1135)



<!-- Sent from Reviewable.io -->

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T16:37:56Z
user: login: ddaspit
body: <img class="emoji" title=":lgtm:" alt=":lgtm:" align="absmiddle" src="https://reviewable.io/lgtm.png" height="20" width="61"/>

> Dismissed @johnml1135 from a discussion.
> *[Reviewable](https://reviewable.io/reviews/sillsdev/serval/670#-:-OOxG-bb6G9p9_Guh5T5:bnfp4nl)* status: <span>:shipit:</span> complete! all files reviewed, all discussions resolved (waiting on @mudiagaobrikisil)



<!-- Sent from Reviewable.io -->

state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-28T16:38:56Z

Review comments:
diff_hunk: @@ -329,13 +329,17 @@ await buildJobService.BuildJobFinishedAsync(
             );
             _logger.LogError("Build faulted ({BuildId}). Error: {ErrorMessage}", buildId, message);
         }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Error during logging: {ex.Message}");
path: src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs
user: login: Copilot
body: Consider replacing Console.WriteLine in the catch block with a call to the logger (or another structured logging mechanism) to maintain logging consistency and improve testability.
```suggestion
            _logger.LogError(ex, "An error occurred during logging for build {BuildId}.", buildId);
```
created_at: 2025-04-10T18:02:22Z
author_association: NONE
diff_hunk: @@ -329,13 +329,17 @@ await buildJobService.BuildJobFinishedAsync(
             );
             _logger.LogError("Build faulted ({BuildId}). Error: {ErrorMessage}", buildId, message);
         }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Error during logging: {ex.Message}");
path: src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs
user: login: johnml1135
body: Actually, I don't think this log is actually needed at all - and it is not clear what is actually happening.  I would remove it.
<!-- Reviewable comment -ONVtB151fNl6WdpPUu0:b-dwqbkw -->
<!-- Sent from Reviewable.io -->

created_at: 2025-04-10T18:10:16Z
author_association: COLLABORATOR
diff_hunk: @@ -329,13 +329,17 @@ await buildJobService.BuildJobFinishedAsync(
             );
             _logger.LogError("Build faulted ({BuildId}). Error: {ErrorMessage}", buildId, message);
         }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Error during logging: {ex.Message}");
path: src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs
user: login: mudiagaobrikisil
body: I have removed it.
<!-- Reviewable comment -ONWBZ3D57h6THI3Jrgs:b-5qf909 -->
<!-- Sent from Reviewable.io -->

created_at: 2025-04-10T19:38:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -329,13 +329,17 @@ await buildJobService.BuildJobFinishedAsync(
             );
             _logger.LogError("Build faulted ({BuildId}). Error: {ErrorMessage}", buildId, message);
         }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Error during logging: {ex.Message}");
path: src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs
user: login: mudiagaobrikisil
body: Was initially for debugging purposes
<!-- Reviewable comment -ONWCiR7D8a8jtAUkFSb:bg05l9l -->
<!-- Sent from Reviewable.io -->

created_at: 2025-04-10T19:39:41Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov-commenter
created_at: 2025-04-09T14:24:43Z
body: ## [Codecov](https://app.codecov.io/gh/sillsdev/serval/pull/670?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sillsdev) Report
:white_check_mark: All modified and coverable lines are covered by tests.
:white_check_mark: Project coverage is 66.08%. Comparing base ([`31d8eb0`](https://app.codecov.io/gh/sillsdev/serval/commit/31d8eb049f67da2663b4506855a0ecdccca59207?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sillsdev)) to head ([`57d329d`](https://app.codecov.io/gh/sillsdev/serval/commit/57d329d5947a7c82e236cc32f723a557f6dfe290?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sillsdev)).
:warning: Report is 151 commits behind head on main.

<details><summary>Additional details and impacted files</summary>



```diff
@@            Coverage Diff             @@
##             main     #670      +/-   ##
==========================================
+ Coverage   64.90%   66.08%   +1.17%     
==========================================
  Files         342      342              
  Lines       18836    18836              
  Branches     2455     2455              
==========================================
+ Hits        12225    12447     +222     
+ Misses       5728     5492     -236     
- Partials      883      897      +14     
```
</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/sillsdev/serval/pull/670?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sillsdev).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=sillsdev).
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
- :package: [JS Bundle Analysis](https://docs.codecov.com/docs/javascript-bundle-analysis): Save yourself from yourself by tracking and limiting bundle sizes in JS merges.
</details>
author_association: NONE
user: login: johnml1135
created_at: 2025-04-10T18:07:17Z
body: 




*[`src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj` line 57 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVrpBgAbNUlFFplkHo:-ONVrpBgAbNUlFFplkHp:bvj0mj2) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Serval/src/Serval.ApiServer/Serval.ApiServer.csproj#L57)):*
> ```XML
> 
> 	<Target Name="NSwag" AfterTargets="PostBuildEvent" Condition=" '$(Configuration)' == 'Debug' ">
> 		<Exec WorkingDirectory="$(ProjectDir)" EnvironmentVariables="ASPNETCORE_ENVIRONMENT=Development;NSWAG=1;DOTNET_ROLL_FORWARD=LatestMajor" Command="$(NSwagExe_Net80) run nswag.json /variables:Configuration=$(Configuration)" />
> ```

Why is this change needed?  I believe we are wanting to stay at .NET 8 for LTS until .NET 10 comes out - https://learn.microsoft.com/en-us/dotnet/core/whats-new/dotnet-10/overview


<!-- Sent from Reviewable.io -->

author_association: COLLABORATOR
user: login: johnml1135
created_at: 2025-04-10T18:07:28Z
body: 




*[`src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs` line 2 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVscyAEc7I_kA-UJHm:-ONVscyAEc7I_kA-UJHn:bdk7ku9) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs#L2)):*
> ```Smalltalk
> [assembly: InternalsVisibleTo("Serval.Machine.Shared.Tests")]
> [assembly: InternalsVisibleTo("Serval.Machine.Shared.Services")]
> ```

Why is this needed?


<!-- Sent from Reviewable.io -->

author_association: COLLABORATOR
user: login: johnml1135
created_at: 2025-04-10T18:09:09Z
body: 




*[`src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs` line 251 at r1](https://reviewable.io/reviews/sillsdev/serval/670#-ONVsjMe7-o_99gHIvPB:-ONVsjMe7-o_99gHIvPC:biecs42) ([raw file](https://github.com/sillsdev/serval/blob/920532ba1c6dbe1ec8acdd50788e2c7e4d4812e8/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs#L251)):*
> ```Smalltalk
>     }
> 
>     internal async Task<bool> TrainJobStartedAsync(
> ```

I am a little confused - I would believe that it would be better to call the top level function and just setup the inputs and outputs so that all lower level functions are called.


<!-- Sent from Reviewable.io -->

author_association: COLLABORATOR
user: login: johnml1135
created_at: 2025-04-22T20:43:55Z
body: @ddaspit - please close any comments of mine that you believe are sufficently dealt with.
author_association: COLLABORATOR