Repository: radius-project/radius

Issue:
Long running test failure: `Test_TerraformRecipe_KubernetesPostgres`: postgres connection refused
### Steps to reproduce

This is a failure in the long-runnning tests on Azure.

```
=== Failed
=== FAIL: test/functional-portable/corerp/noncloud/resources Test_TerraformRecipe_KubernetesPostgres/deploy_testdata/corerp-resources-terraform-postgres.bicep (43.41s)
    rptest.go:391: running step 0 of 1: deploy testdata/corerp-resources-terraform-postgres.bicep
    deployexecutor.go:80: deploying deploy testdata/corerp-resources-terraform-postgres.bicep from file testdata/corerp-resources-terraform-postgres.bicep
    cli.go:394: [heartbeat] command rad deploy /home/runner/work/radius/radius/test/functional-portable/corerp/noncloud/resources/testdata/corerp-resources-terraform-postgres.bicep --parameters moduleServer=http://tf-module-server.radius-test-tf-module-server.svc.cluster.local --parameters userName=postgres --parameters ***' is still running after 10.004271377s
    cli.go:394: [heartbeat] command rad deploy /home/runner/work/radius/radius/test/functional-portable/corerp/noncloud/resources/testdata/corerp-resources-terraform-postgres.bicep --parameters moduleServer=http://tf-module-server.radius-test-tf-module-server.svc.cluster.local --parameters userName=postgres --parameters ***' is still running after 20.004521106s
    cli.go:394: [heartbeat] command rad deploy /home/runner/work/radius/radius/test/functional-portable/corerp/noncloud/resources/testdata/corerp-resources-terraform-postgres.bicep --parameters moduleServer=http://tf-module-server.radius-test-tf-module-server.svc.cluster.local --parameters userName=postgres --parameters ***' is still running after 30.004712234s
    cli.go:394: [heartbeat] command rad deploy /home/runner/work/radius/radius/test/functional-portable/corerp/noncloud/resources/testdata/corerp-resources-terraform-postgres.bicep --parameters moduleServer=http://tf-module-server.radius-test-tf-module-server.svc.cluster.local --parameters userName=postgres --parameters ***' is still running after 40.005097403s
    cli.go:341: [rad] Building /home/runner/work/radius/radius/test/functional-portable/corerp/noncloud/resources/testdata/corerp-resources-terraform-postgres.bicep...
    cli.go:341: [rad] WARNING: The following experimental Bicep features have been enabled: Extensibility. Experimental features should be enabled for testing purposes only, as there are no guarantees about the quality or stability of these features. Do not enable these settings for any production usage, or your production environment may be subject to breaking.
    cli.go:341: [rad] Deploying template '/home/runner/work/radius/radius/test/functional-portable/corerp/noncloud/resources/testdata/corerp-resources-terraform-postgres.bicep' into environment '/planes/radius/local/resourceGroups/kind-radius/providers/Applications.Core/environments/kind-radius' from workspace 'radlrtest00-aks-admin'...
    cli.go:341: [rad] 
    cli.go:341: [rad] Deployment In Progress...
    cli.go:341: [rad] 
    cli.go:341: [rad] Error: {
    cli.go:341: [rad]   "code": "DeploymentFailed",
    cli.go:341: [rad]   "message": "At least one resource deployment operation failed. Please see the details for the specific operation that failed.",
    cli.go:341: [rad]   "target": "/planes/radius/local/resourceGroups/kind-radius/providers/Microsoft.Resources/deployments/rad-deploy-c94310b2-7e8d-488c-93c6-d6d45b8abc52",
    cli.go:341: [rad]   "details": [
    cli.go:341: [rad]     {
    cli.go:341: [rad]       "code": "OK",
    cli.go:341: [rad]       "message": "",
    cli.go:341: [rad]       "target": "/planes/radius/local/resourceGroups/kind-radius/providers/Applications.Core/applications/corerp-resources-terraform-pg-app"
    cli.go:341: [rad]     },
    cli.go:341: [rad]     {
    cli.go:341: [rad]       "code": "OK",
    cli.go:341: [rad]       "message": "",
    cli.go:341: [rad]       "target": "/planes/radius/local/resourceGroups/kind-radius/providers/Applications.Core/environments/corerp-resources-terraform-pg-env"
    cli.go:341: [rad]     },
    cli.go:341: [rad]     {
    cli.go:341: [rad]       "code": "OK",
    cli.go:341: [rad]       "message": "",
    cli.go:341: [rad]       "target": "/planes/radius/local/resourceGroups/kind-radius/providers/Applications.Core/secretStores/pgs-secretstore"
    cli.go:341: [rad]     },
    cli.go:341: [rad]     {
    cli.go:341: [rad]       "code": "ResourceDeploymentFailure",
    cli.go:341: [rad]       "message": "Failed",
    cli.go:341: [rad]       "target": "/planes/radius/local/resourceGroups/kind-radius/providers/Applications.Core/extenders/pgs-resources-terraform-pgsapp",
    cli.go:341: [rad]       "details": [
    cli.go:341: [rad]         {
    cli.go:341: [rad]           "code": "RecipeDeploymentFailed",
    cli.go:341: [rad]           "message": "terraform apply failure: exit status 1\n\nError: Error connecting to PostgreSQL server postgres.corerp-resources-terraform-pg-app.svc.cluster.local (scheme: postgres): dial tcp 172.16.198.233:5432: connect: connection refused\n\n  with module.defaultpostgres.postgresql_database.pg_db_test,\n  on .terraform/modules/defaultpostgres/main.tf line 77, in resource \"postgresql_database\" \"pg_db_test\":\n  77: resource postgresql_database \"pg_db_test\" {\n\n"
    cli.go:341: [rad]         }
    cli.go:341: [rad]       ]
    cli.go:341: [rad]     }
    cli.go:341: [rad]   ]
    cli.go:341: [rad] }
    cli.go:341: [rad] 
    cli.go:341: [rad] TraceId:  a90b09dc7e604b7da6e65d09c85f[79](https://github.com/radius-project/radius/actions/runs/13550383259/job/37872354388#step:25:80)87
    cli.go:341: [rad] 
    cli.go:341: [rad] 
    deployexecutor.go:83: 
        	Error Trace:	/home/runner/work/radius/radius/test/step/deployexecutor.go:83
        	            				/home/runner/work/radius/radius/test/rp/rptest.go:392
        	Error:      	Received unexpected error:
        	            	code DeploymentFailed: err At least one resource deployment operation failed. Please see the details for the specific operation that failed.
        	Test:       	Test_TerraformRecipe_KubernetesPostgres/deploy_testdata/corerp-resources-terraform-postgres.bicep
        	Messages:   	failed to deploy deploy testdata/corerp-resources-terraform-postgres.bicep
    --- FAIL: Test_TerraformRecipe_KubernetesPostgres/deploy_testdata/corerp-resources-terraform-postgres.bicep (43.41s)
```

### Observed behavior

_No response_

### Desired behavior

The test should pass

### Workaround

_No response_

### rad Version

Long-running tests

### Operating system

_No response_

### Additional context

_No response_

### Would you like to support us?

- [x] Yes, I would like to support you

[AB#14698](https://dev.azure.com/azure-octo/e61041b4-555f-47ae-95b2-4f8ab480ea57/_workitems/edit/14698)

Hints from issue comments:
@radius-triage-bot: :wave: @willdavsmith Thanks for filing this bug report. 

A project maintainer will review this report and get back to you soon. If you'd like immediate help troubleshooting, please visit our [Discord server](https://aka.ms/radius/discord).

_For more information on our triage process please visit our [triage overview](https://github.com/radius-project/radius/blob/main/docs/contributing/triage/triage-process.md)_@lakshmimsft: This issue was resolved with updating image of AKS nodes and restarting nodes.

![Image](https://github.com/user-attachments/assets/d92ca0a1-8e5c-4c71-bae4-f74dd398dfe9)

Solution patch:
diff --git a/.github/scripts/cleanup-long-running-cluster.sh b/.github/scripts/cleanup-long-running-cluster.sh
index e9d9e79e8..e3f80b219 100755
--- a/.github/scripts/cleanup-long-running-cluster.sh
+++ b/.github/scripts/cleanup-long-running-cluster.sh
@@ -49,13 +49,16 @@ fi
 
 # Delete all test namespaces.
 echo "delete all test namespaces"
-namespaces=$(kubectl get namespace |
-    grep -E '^kubernetes-interop-tutorial.*|^corerp.*|^test.*|^default-.*|^radiusfunctionaltestbucket.*|^radius-test.*|^kubernetes-cli.*|^dpsb-.*|^dsrp-.*|^azstorage-workload.*|^dapr-serviceinvocation|^daprrp-rs-.*|^dapr-sts-.*|^mynamespace.*|^demo.*|^tutorial-demo.*|^ms.+' |
-    awk '{print $1}')
+namespace_whitelist=("cert-manager" "dapr-system" "default" "gatekeeper-system" "kube-node-lease" "kube-public" "kube-system" "radius-system")
+namespaces=$(kubectl get namespaces --no-headers -o custom-columns=":metadata.name")
 for ns in $namespaces; do
     if [ -z "$ns" ]; then
         break
     fi
-    echo "deleting namespaces: $ns"
-    kubectl delete namespace $ns --ignore-not-found=true
+    if [[ " ${namespace_whitelist[@]} " =~ " ${ns} " ]]; then
+        echo "skip deletion: $ns"
+    else
+        echo "deleting namespaces: $ns"
+        kubectl delete namespace $ns --ignore-not-found=true
+    fi
 done
diff --git a/.github/workflows/long-running-azure.yaml b/.github/workflows/long-running-azure.yaml
index b78ba6753..112f0ef0b 100644
--- a/.github/workflows/long-running-azure.yaml
+++ b/.github/workflows/long-running-azure.yaml
@@ -48,10 +48,13 @@ on:
   workflow_dispatch:
     inputs:
       skip-build:
-        description: 'Skip build (true/false)'
+        description: 'Skip build (true/false). Setting to false will cause the tests to run with the latest changes from the main branch.'
         required: false
         default: 'true'
-        type: string
+        type: choice
+        options:
+          - 'true'
+          - 'false'
   schedule:
     # Run every 2 hours
     - cron: "0 */2 * * *"



Test patch:
diff --git a/test/infra/azure/main.bicep b/test/infra/azure/main.bicep
index 963ffcad7..4703d0897 100644
--- a/test/infra/azure/main.bicep
+++ b/test/infra/azure/main.bicep
@@ -90,7 +90,7 @@ module aksCluster './modules/akscluster.bicep' = {
   params: {
     name: aksClusterName
     location: location
-    kubernetesVersion: '1.28.5'
+    kubernetesVersion: '1.31.5'
     logAnalyticsWorkspaceId: logAnalyticsWorkspace.outputs.id
     systemAgentPoolName: 'agentpool'
     systemAgentPoolVmSize: 'Standard_D4as_v5'