Repository: Ed-Fi-Alliance-OSS/Data-Management-Service
Pull request #490: [DMS-593] Running Keycloak first in DMS Docker
Author: login: semalaiappan
Created: 2025-04-17T23:47:30Z | Merged: 2025-04-18T21:27:13Z



Diff:
diff --git a/.github/workflows/on-config-pullrequest.yml b/.github/workflows/on-config-pullrequest.yml
index dc4409a55..892637d51 100644
--- a/.github/workflows/on-config-pullrequest.yml
+++ b/.github/workflows/on-config-pullrequest.yml
@@ -150,7 +150,7 @@ jobs:
 
       - name: Run End to End Tests
         if: success()
-        run: ./build-config.ps1 E2ETest
+        run: ./build-config.ps1 E2ETest  -Configuration Release
 
       - name: Upload Logs
         if: failure()
diff --git a/.github/workflows/on-dms-pullrequest.yml b/.github/workflows/on-dms-pullrequest.yml
index dd7251ecc..744cc561f 100644
--- a/.github/workflows/on-dms-pullrequest.yml
+++ b/.github/workflows/on-dms-pullrequest.yml
@@ -152,7 +152,7 @@ jobs:
 
       - name: Run OpenSearch End to End Tests
         if: success()
-        run: ./build-dms.ps1 E2ETest -EnableOpenSearch
+        run: ./build-dms.ps1 E2ETest -EnableOpenSearch  -Configuration Release
 
       - name: Upload Logs
         if: failure()
diff --git a/.github/workflows/scheduled-build.yml b/.github/workflows/scheduled-build.yml
index cc09d49ca..e9cf6fd4b 100644
--- a/.github/workflows/scheduled-build.yml
+++ b/.github/workflows/scheduled-build.yml
@@ -51,11 +51,11 @@ jobs:
 
       - name: Run ElasticSearch End to End Tests
         if: success()
-        run: ./build-dms.ps1 E2ETest -EnableElasticSearch
+        run: ./build-dms.ps1 E2ETest -EnableElasticSearch -Configuration Release
 
       - name: Run OpenSearch End to End Tests
         if: success()
-        run: ./build-dms.ps1 E2ETest -EnableOpenSearch
+        run: ./build-dms.ps1 E2ETest -EnableOpenSearch  -Configuration Release
 
       - name: Notify Slack on success
         if: success()
diff --git a/.github/workflows/scheduled-pre-image-test.yml b/.github/workflows/scheduled-pre-image-test.yml
index 2da96a26b..72cd261eb 100644
--- a/.github/workflows/scheduled-pre-image-test.yml
+++ b/.github/workflows/scheduled-pre-image-test.yml
@@ -42,11 +42,11 @@ jobs:
 
       - name: Run OpenSearch End to End Tests
         if: success()
-        run: ./build-dms.ps1 E2ETest -EnableOpenSearch -UsePublishedImage
+        run: ./build-dms.ps1 E2ETest -EnableOpenSearch -UsePublishedImage -Configuration Release
 
       - name: Run ElasticSearch End to End Tests
         if: success()
-        run: ./build-dms.ps1 E2ETest -EnableElasticSearch -UsePublishedImage
+        run: ./build-dms.ps1 E2ETest -EnableElasticSearch -UsePublishedImage -Configuration Release
 
       - name: Notify Slack on success
         if: success()
diff --git a/eng/docker-compose/start-local-dms.ps1 b/eng/docker-compose/start-local-dms.ps1
index 11dad0083..c6190dd50 100644
--- a/eng/docker-compose/start-local-dms.ps1
+++ b/eng/docker-compose/start-local-dms.ps1
@@ -39,9 +39,7 @@ $files = @(
     "-f",
     "postgresql.yml",
     "-f",
-    "local-dms.yml",
-    "-f",
-    "keycloak.yml"
+    "local-dms.yml"
 )
 
 if ($SearchEngine -eq "ElasticSearch") {
@@ -82,17 +80,16 @@ else {
     )
     if ($r) { $upArgs += @("--build") }
 
-    Write-Output "Starting locally-built DMS"
-
-    docker compose $files --env-file $EnvironmentFile -p dms-local up $upArgs
-
+    Write-Output "Starting Keycloak first..."
+    docker compose -f keycloak.yml --env-file $EnvironmentFile -p dms-local up $upArgs
     if ($LASTEXITCODE -ne 0) {
-        throw "Unable to start local Docker environment, with exit code $LASTEXITCODE."
+        throw "Failed to start Keycloak. Exit code $LASTEXITCODE"
     }
 
+    Write-Output "Waiting for Keycloak to initialize..."
     Start-Sleep 20
-    ./setup-connectors.ps1 $EnvironmentFile $SearchEngine
 
+    Write-Output "Running setup-keycloak.ps1 scripts..."
     Start-Sleep 5
     # Create client with default edfi_admin_api/full_access scope
     ./setup-keycloak.ps1
@@ -102,4 +99,18 @@ else {
 
     # Create client with edfi_admin_api/authMetadata_readonly_access scope
     ./setup-keycloak.ps1 -NewClientId "CMSAuthMetadataReadOnlyAccess" -NewClientName "CMS Auth Endpoints Only Access" -ClientScopeName "edfi_admin_api/authMetadata_readonly_access"
+
+    Write-Output "Starting locally-built DMS"
+
+    docker compose $files --env-file $EnvironmentFile -p dms-local up $upArgs
+
+    if ($LASTEXITCODE -ne 0) {
+        throw "Unable to start local Docker environment, with exit code $LASTEXITCODE."
+    }
+
+    Start-Sleep 20
+
+    Write-Output "Running connector setup..."
+    ./setup-connectors.ps1 $EnvironmentFile $SearchEngine
+
 }
diff --git a/eng/docker-compose/start-published-dms.ps1 b/eng/docker-compose/start-published-dms.ps1
index d130666da..63a4514e1 100644
--- a/eng/docker-compose/start-published-dms.ps1
+++ b/eng/docker-compose/start-published-dms.ps1
@@ -35,9 +35,7 @@ $files = @(
     "-f",
     "postgresql.yml",
     "-f",
-    "published-dms.yml",
-    "-f",
-    "keycloak.yml"
+    "published-dms.yml"
 )
 
 if ($SearchEngine -eq "ElasticSearch") {
@@ -73,16 +71,16 @@ else {
         docker network create dms
     }
 
-    Write-Output "Starting published DMS"
-    docker compose $files --env-file $EnvironmentFile -p dms-published up -d
-
+    Write-Output "Starting Keycloak first..."
+    docker compose -f keycloak.yml -env-file $EnvironmentFile -p dms-published up -d
     if ($LASTEXITCODE -ne 0) {
-        throw "Unable to start Published Docker environment, with exit code $LASTEXITCODE."
+        throw "Failed to start Keycloak. Exit code $LASTEXITCODE"
     }
 
+    Write-Output "Waiting for Keycloak to initialize..."
     Start-Sleep 20
-    ./setup-connectors.ps1 $EnvironmentFile $SearchEngine
 
+    Write-Output "Running setup-keycloak.ps1 scripts..."
     Start-Sleep 5
     # Create client with default edfi_admin_api/full_access scope
     ./setup-keycloak.ps1
@@ -92,4 +90,15 @@ else {
 
     # Create client with edfi_admin_api/authMetadata_readonly_access scope
     ./setup-keycloak.ps1 -NewClientId "CMSAuthMetadataReadOnlyAccess" -NewClientName "CMS Auth Endpoints Only Access" -ClientScopeName "edfi_admin_api/authMetadata_readonly_access"
+
+    Write-Output "Starting published DMS"
+    docker compose $files --env-file $EnvironmentFile -p dms-published up -d
+
+    if ($LASTEXITCODE -ne 0) {
+        throw "Unable to start Published Docker environment, with exit code $LASTEXITCODE."
+    }
+
+    Start-Sleep 20
+    ./setup-connectors.ps1 $EnvironmentFile $SearchEngine
+
 }
diff --git a/src/dms/frontend/EdFi.DataManagementService.Frontend.SchoolYearLoader/Program.cs b/src/dms/frontend/EdFi.DataManagementService.Frontend.SchoolYearLoader/Program.cs
index 85a8c9a08..123bdbfdb 100644
--- a/src/dms/frontend/EdFi.DataManagementService.Frontend.SchoolYearLoader/Program.cs
+++ b/src/dms/frontend/EdFi.DataManagementService.Frontend.SchoolYearLoader/Program.cs
@@ -63,6 +63,9 @@ await result.WithParsedAsync(async options =>
                         logger.LogCritical("Error: CurrentSchoolYear must be a positive integer.");
                         throw new InvalidOperationException("Error: CurrentSchoolYear must be a positive integer.");
                     }
+
+                    await host.StartAsync();
+
                     IConfiguration config = host.Services.GetRequiredService<IConfiguration>();
 
                     var apiService = host.Services.GetRequiredService<IApiService>();
@@ -78,7 +81,7 @@ await result.WithParsedAsync(async options =>
 
                     await SchoolYearProcessor.ProcessSchoolYearTypesAsync(logger, apiService, options.StartYear, options.EndYear, options.CurrentSchoolYear);
                 });
-                await host.RunAsync();
+                await host.StopAsync();
                 return 0;
             }
             catch (Exception ex)
diff --git a/src/dms/run.sh b/src/dms/run.sh
index 6a09075c9..a72b0d032 100644
--- a/src/dms/run.sh
+++ b/src/dms/run.sh
@@ -48,7 +48,8 @@ if [ "$USE_API_SCHEMA_PATH" = true ]; then
     dotnet /app/ApiSchemaDownloader/EdFi.DataManagementService.ApiSchemaDownloader.dll -p ${HOMOGRAPH_PACKAGE} -d ${API_SCHEMA_PATH} -v ${HOMOGRAPH_PACKAGE_VERSION}
 fi
 
-echo "Running EdFi.DataManagementService.Frontend.SchoolYearLoader Cli START_YEAR=${START_YEAR}  END_YEAR=${END_YEAR} CURRENT_SCHOOL_YEAR=${CURRENT_SCHOOL_YEAR"
+echo "Running EdFi.DataManagementService.Frontend.SchoolYearLoader Cli START_YEAR=${START_YEAR}  END_YEAR=${END_YEAR} CURRENT_SCHOOL_YEAR=${CURRENT_SCHOOL_YEAR}..."
 dotnet /app/SchoolYearLoader/EdFi.DataManagementService.Frontend.SchoolYearLoader.dll --startYear ${START_YEAR} --endYear ${END_YEAR} --currentSchoolYear  ${CURRENT_SCHOOL_YEAR}
 
+echo "Running EdFi.DataManagementService.Frontend.AspNetCore..."
 dotnet EdFi.DataManagementService.Frontend.AspNetCore.dll


Reviews:
user: login: CSR2017
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T14:54:35Z
user: login: CSR2017
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T16:37:47Z
user: login: semalaiappan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T17:18:38Z
user: login: semalaiappan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T17:56:30Z
user: login: CSR2017
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T21:10:06Z
user: login: vimayya
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T21:26:35Z

Review comments:
diff_hunk: @@ -73,16 +71,16 @@ else {
         docker network create dms
     }
 
-    Write-Output "Starting published DMS"
-    docker compose $files --env-file $EnvironmentFile -p dms-published up -d
-
+    Write-Output "Starting Keycloak first..."
+    docker compose -f keycloak.yml -env-file $EnvironmentFile -p dms-published up -d
     if ($LASTEXITCODE -ne 0) {
-        throw "Unable to start Published Docker environment, with exit code $LASTEXITCODE."
+        throw "Failed to start Keycloak. Exit code $LASTEXITCODE"
     }
 
+    Write-Output "Waiting for Keycloak to initialize..."
     Start-Sleep 20
path: eng/docker-compose/start-published-dms.ps1
user: login: CSR2017
body: @semalaiappan 
The sleep time increased to 45 seconds now. Please reduce the wait time for keycloak setup and try. Same for the start-local-dms as well.
created_at: 2025-04-18T14:47:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -102,4 +99,18 @@ else {
 
     # Create client with edfi_admin_api/authMetadata_readonly_access scope
     ./setup-keycloak.ps1 -NewClientId "CMSAuthMetadataReadOnlyAccess" -NewClientName "CMS Auth Endpoints Only Access" -ClientScopeName "edfi_admin_api/authMetadata_readonly_access"
+
path: eng/docker-compose/start-local-dms.ps1
user: login: CSR2017
body: Nice simple fix for the issue 👍 
created_at: 2025-04-18T16:37:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -73,16 +71,16 @@ else {
         docker network create dms
     }
 
-    Write-Output "Starting published DMS"
-    docker compose $files --env-file $EnvironmentFile -p dms-published up -d
-
+    Write-Output "Starting Keycloak first..."
+    docker compose -f keycloak.yml -env-file $EnvironmentFile -p dms-published up -d
     if ($LASTEXITCODE -ne 0) {
-        throw "Unable to start Published Docker environment, with exit code $LASTEXITCODE."
+        throw "Failed to start Keycloak. Exit code $LASTEXITCODE"
     }
 
+    Write-Output "Waiting for Keycloak to initialize..."
     Start-Sleep 20
path: eng/docker-compose/start-published-dms.ps1
user: login: semalaiappan
body: ![image](https://github.com/user-attachments/assets/5480baaa-b257-4049-a340-0b8c3de4581e)

created_at: 2025-04-18T17:18:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -73,16 +71,16 @@ else {
         docker network create dms
     }
 
-    Write-Output "Starting published DMS"
-    docker compose $files --env-file $EnvironmentFile -p dms-published up -d
-
+    Write-Output "Starting Keycloak first..."
+    docker compose -f keycloak.yml -env-file $EnvironmentFile -p dms-published up -d
     if ($LASTEXITCODE -ne 0) {
-        throw "Unable to start Published Docker environment, with exit code $LASTEXITCODE."
+        throw "Failed to start Keycloak. Exit code $LASTEXITCODE"
     }
 
+    Write-Output "Waiting for Keycloak to initialize..."
     Start-Sleep 20
path: eng/docker-compose/start-published-dms.ps1
user: login: semalaiappan
body: I am getting this error if i reduce sleep time
created_at: 2025-04-18T17:56:30Z
author_association: CONTRIBUTOR

PR comments:
user: login: CSR2017
created_at: 2025-04-18T14:52:59Z
body: @semalaiappan 
Please add an end-to-end (E2E) test for CRUD operations on the schoolYearTypes endpoint.
As part of the test, try to read a specific school year that falls between the configured minimum and maximum year values. This will help ensure the CLI application is functioning as expected.
author_association: CONTRIBUTOR
user: login: semalaiappan
created_at: 2025-04-18T19:43:57Z
body: @CSR2017  I will do end-to-end (E2E) test for SchoolYearType in another PR . Since it's breaking current code . let merge this PR once you test it today 
author_association: CONTRIBUTOR