Repository: NVIDIA/TensorRT-LLM
Pull request #3091: chore: Blossom debug hook
Author: login: yiqingy0
Created: 2025-03-26T06:47:30Z | Merged: 2025-04-07T07:47:48Z

Blossom debug hook


Diff:
diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy
index 7f5b25d1d4b7..f2560d90b81f 100644
--- a/jenkins/L0_MergeRequest.groovy
+++ b/jenkins/L0_MergeRequest.groovy
@@ -94,6 +94,8 @@ def EXTRA_STAGE_LIST = "extra_stage"
 def MULTI_GPU_FILE_CHANGED = "multi_gpu_file_changed"
 @Field
 def ONLY_PYTORCH_FILE_CHANGED = "only_pytorch_file_changed"
+@Field
+def DEBUG_MODE = "debug"
 
 def testFilter = [
     (REUSE_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get(REUSE_STAGE_LIST, null)?.tokenize(',')),
@@ -107,6 +109,7 @@ def testFilter = [
     (EXTRA_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get((EXTRA_STAGE_LIST), null)?.tokenize(',')),
     (MULTI_GPU_FILE_CHANGED): false,
     (ONLY_PYTORCH_FILE_CHANGED): false,
+    (DEBUG_MODE): gitlabParamsFromBot.get(DEBUG_MODE, false),
 ]
 
 String reuseBuild = gitlabParamsFromBot.get('reuse_build', null)
diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 01df872e8502..caf754d76b33 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -114,7 +114,9 @@ def EXTRA_STAGE_LIST = "extra_stage"
 def MULTI_GPU_FILE_CHANGED = "multi_gpu_file_changed"
 @Field
 def ONLY_PYTORCH_FILE_CHANGED = "only_pytorch_file_changed"
-
+@Field
+def DEBUG_MODE = "debug"
+@Field
 def testFilter = [
     (REUSE_STAGE_LIST): null,
     (ENABLE_SKIP_TEST): false,
@@ -127,6 +129,7 @@ def testFilter = [
     (EXTRA_STAGE_LIST): null,
     (MULTI_GPU_FILE_CHANGED): false,
     (ONLY_PYTORCH_FILE_CHANGED): false,
+    (DEBUG_MODE): false,
 ]
 
 String getShortenedJobName(String path)
@@ -607,6 +610,66 @@ def renderTestDB(testContext, llmSrc, stageName) {
     return testList
 }
 
+def getSSHConnectionPorts(portConfigFile, stageName)
+{
+    def type = stageName.split('-')[0]
+    echo "The type is: ${type}"
+    def fileContent = sh(script: "cat ${portConfigFile}", returnStdout: true).trim()
+
+    // Get available VM port list from portConfigFile based on stage name (e.g. A10: [10022, 10023])
+    def portList = []
+    fileContent.split('\n').each { line ->
+        def matcher = (line =~ /(.+?)=\[(.+?)\]/)
+        if (matcher) {
+            def key = matcher[0][1].replaceAll("\\s","")
+            def values = matcher[0][2].replaceAll("\\s","").split(',').collect { it.replaceAll("\\s","") }
+            if (key == type) {
+                portList.addAll(values)
+            }
+        }
+    }
+    echo "Port List for ${type}: ${portList}"
+
+    // Get current port usage status
+    def portUsage = ""
+    withCredentials([
+        usernamePassword(credentialsId: 'tensorrt_llm_infra_debug_vm_01_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD'),
+        string(credentialsId: 'DEBUG_HOST_NAME', variable: 'HOST_NAME')
+        ]) {
+        portUsage = sh(script: "ssh -v ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'netstat -tuln'",returnStdout: true)
+    }
+    echo "Port Usage: ${portUsage}"
+
+    // Get an available VM port
+    def userPort = 0
+    while (portList.size() > 0) {
+        def randomIndex = (int)(Math.random() * portList.size())
+        def curPort = portList[randomIndex].toInteger()
+        if (!portUsage.contains(":${curPort}")) {
+            userPort = curPort
+            break
+        }
+        portList.remove(randomIndex)
+    }
+
+    if (userPort == 0) {
+        echo "There is no available port for ${type}"
+        return [0, 0]
+    }
+
+    echo "The chosen port is: ${userPort}"
+
+    // Calculate autossh monitor port by subtracting 9000 from VM port (e.g. 10022 -> 1022)
+    // If monitor port is already in use, randomly assign a value between 2000-3000
+    def monitorPort = userPort - 9000
+    while (portUsage.contains(":${monitorPort}")) {
+        monitorPort = 2000 + (int)(Math.random() * 1000)
+    }
+
+    echo "The monitor port is: ${monitorPort}"
+
+    return [userPort, monitorPort]
+}
 
 def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312")
 {
@@ -668,6 +731,77 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         trtllm_utils.llmExecStepWithRetry(pipeline, script: "git config --global --add safe.directory \"*\"")
     }
 
+    if (testFilter[(DEBUG_MODE)]) {
+        stage("Interactive debug session")
+        {
+            testFilter[(DEBUG_MODE)] = false
+
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install openssh-server -y")
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install autossh -y")
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install sshpass -y")
+
+            sh """
+                echo 'Port 22' >> /etc/ssh/sshd_config
+                echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config
+                echo 'PasswordAuthentication yes' >> /etc/ssh/sshd_config
+                echo 'PubkeyAuthentication yes' >> /etc/ssh/sshd_config
+                echo 'AllowTcpForwarding yes' >> /etc/ssh/sshd_config
+                echo 'GatewayPorts yes' >> /etc/ssh/sshd_config
+                cat /etc/ssh/sshd_config
+            """
+
+            sh "service ssh restart"
+            sh "service ssh status"
+
+            sh "ssh-keygen -t rsa -b 2048 -f ~/.ssh/id_rsa -N '' -q"
+
+            sh """
+                chmod 700 ~/.ssh
+                chmod 400 ~/.ssh/id_rsa
+                touch ~/.ssh/authorized_keys
+                chmod 600 ~/.ssh/authorized_keys
+            """
+
+            // The portConfig file is in the VM
+            def portConfigFilePath = "/root/.ssh/ports_config.txt"
+
+            withCredentials([
+                usernamePassword(credentialsId: 'tensorrt_llm_infra_debug_vm_01_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD'),
+                string(credentialsId: 'DEBUG_HOST_NAME', variable: 'HOST_NAME')
+                ]) {
+                sh "sshpass -p ${PASSWORD} -v ssh ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'cat >> ~/.ssh/authorized_keys' < ~/.ssh/id_rsa.pub"
+                sh "ssh -v ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'echo \"\" > ~/.ssh/known_hosts && cat ~/.ssh/id_rsa.pub' >> ~/.ssh/authorized_keys"
+                sh "ssh -v ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'cat ~/.ssh/ports_config.txt' >> ${portConfigFilePath}"
+
+                def (int userPort, int monitorPort) = getSSHConnectionPorts(portConfigFilePath, stageName)
+                if (userPort == 0) {
+                    echo "Fail to setup an interactive debug session and exit the debug mode."
+                    testFilter[(DEBUG_MODE)] = false
+                    return
+                }
+
+                sh "ssh -f -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -L 1111:127.0.0.1:${monitorPort} -R ${monitorPort}:127.0.0.1:1112 -NR ${userPort}:localhost:22 ${USERNAME}@${HOST_NAME}"
+                sh "autossh -fNR ${userPort}:localhost:22 ${USERNAME}@${HOST_NAME}"
+                sh "ps aux | grep ssh"
+                try {
+                    timeout(time: 2, unit: 'HOURS') {
+                        input message: "Pause 2 hours for Pre-Debug. Please type 'ssh root@${HOST_NAME} -p ${userPort}' on the CLI to create the connection. Please press the button to proceed when you finish debugging."
+                    }
+                } catch (InterruptedException e) {
+                    echo "Pre-debug session was interrupted by user or timeout"
+                    currentBuild.result = 'ABORTED'
+                    error("Pipeline aborted during pre-debug session")
+                } catch (Exception e) {
+                    echo "An error occurred during pre-debug session: ${e.message}"
+                    currentBuild.result = 'FAILURE'
+                    error("Error in pre-debug session: ${e.message}")
+                }
+            }
+
+            testFilter[(DEBUG_MODE)] = true
+        }
+    }
+
     stage ("[${stageName}] Run Pytest")
     {
         echoNodeAndGpuInfo(pipeline, stageName)
@@ -780,6 +914,21 @@ def runLLMTestlistOnPlatform(pipeline, platform, testList, config=VANILLA_CONFIG
     cacheErrorAndUploadResult(stageName, {
         runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver)
     }, {
+        if (testFilter[(DEBUG_MODE)]) {
+            try {
+                timeout(time: 2, unit: 'HOURS') {
+                    input message: "Pause 2 hours for Post-Debug. Please press the button to proceed when you finish debugging."
+                }
+            } catch (InterruptedException e) {
+                echo "Post-debug session was interrupted by user or timeout"
+                currentBuild.result = 'ABORTED'
+                error("Pipeline aborted during post-debug session")
+            } catch (Exception e) {
+                echo "An error occurred during post-debug session: ${e.message}"
+                currentBuild.result = 'FAILURE'
+                error("Error in post-debug session: ${e.message}")
+            }
+        }
         def llmPath = sh (script: "realpath .", returnStdout: true).trim()
         def llmSrc = "${llmPath}/${LLM_ROOT}${config}/TensorRT-LLM/src"
         // CPP tests will generate test result in ${llmSrc}/cpp/build_backup/, move these files to job result folder


Reviews:
user: login: chzblych
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-02T12:39:47Z
user: login: chzblych
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-02T12:39:50Z
user: login: yiqingy0
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T06:17:22Z

Review comments:
diff_hunk: @@ -679,6 +742,77 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         trtllm_utils.llmExecStepWithRetry(pipeline, script: "git config --global --add safe.directory \"*\"")
     }
 
+    if (testFilter[(DEBUG_MODE)]) {
+        stage("Interactive debug session")
+        {
+            testFilter[(DEBUG_MODE)] = false
+
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install openssh-server -y")
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install autossh -y")
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install sshpass -y")
+
+            sh """
+                echo 'Port 22' >> /etc/ssh/sshd_config
+                echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config
+                echo 'PasswordAuthentication yes' >> /etc/ssh/sshd_config
+                echo 'PubkeyAuthentication yes' >> /etc/ssh/sshd_config
+                echo 'AllowTcpForwarding yes' >> /etc/ssh/sshd_config
+                echo 'GatewayPorts yes' >> /etc/ssh/sshd_config
+                cat /etc/ssh/sshd_config
+            """
+
+            sh "service ssh restart"
+            sh "service ssh status"
+
+            sh "ssh-keygen -t rsa -b 2048 -f ~/.ssh/id_rsa -N '' -q"
+
+            sh """
+                chmod 700 ~/.ssh
+                chmod 400 ~/.ssh/id_rsa
+                touch ~/.ssh/authorized_keys
+                chmod 600 ~/.ssh/authorized_keys
+            """
+
+            // The portConfig file is in the VM
+            def portConfigFilePath = "/root/.ssh/ports_config.txt"
+
+            withCredentials([
+                usernamePassword(credentialsId: 'tensorrt_llm_infra_debug_vm_01_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD'),
+                string(credentialsId: 'DEBUG_HOST_NAME', variable: 'HOST_NAME')
+                ]) {
+                sh "sshpass -p ${PASSWORD} -v ssh ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'cat >> ~/.ssh/authorized_keys' < ~/.ssh/id_rsa.pub"
+                sh "ssh -v ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'echo \"\" > ~/.ssh/known_hosts && cat ~/.ssh/id_rsa.pub' >> ~/.ssh/authorized_keys"
+                sh "ssh -v ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'cat ~/.ssh/ports_config.txt' >> ${portConfigFilePath}"
+
+                def (int userPort, int monitorPort) = getSSHConnectionPorts(portConfigFilePath, stageName)
+                if (userPort == 0) {
+                    echo "Fail to setup an interactive debug session and exit the debug mode."
+                    testFilter[(DEBUG_MODE)] = false
+                    return
+                }
+
+                sh "ssh -f -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -L 1111:127.0.0.1:${monitorPort} -R ${monitorPort}:127.0.0.1:1112 -NR ${userPort}:localhost:22 ${USERNAME}@${HOST_NAME}"
+                sh "autossh -fNR ${userPort}:localhost:22 ${USERNAME}@${HOST_NAME}"
+                sh "ps aux | grep ssh"
+                try {
+                    timeout(time: 2, unit: 'HOURS') {
+                        input message: "Pause 2 hours for Pre-Debug. Please type 'ssh root@${HOST_NAME} -p ${userPort}' on the CLI to create the connection. Please press the button to proceed when you finish debugging."
+                    }
+                } catch (InterruptedException e) {
+                    echo "Pre-debug session was interrupted by user or timeout"
+                    currentBuild.result = 'ABORTED'
+                    error("Pipeline aborted during pre-debug session")
+                } catch (Exception e) {
+                    echo "An error occurred during pre-debug session: ${e.message}"
+                    currentBuild.result = 'FAILURE'
+                    error("Error in pre-debug session: ${e.message}")
+                }
+            }
+
+            testFilter[(DEBUG_MODE)] = true
path: jenkins/L0_Test.groovy
user: login: chzblych
body: Why does it need to reset `testFilter[(DEBUG_MODE)]` to `true`?
created_at: 2025-04-02T12:39:46Z
author_association: COLLABORATOR
diff_hunk: @@ -679,6 +742,77 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         trtllm_utils.llmExecStepWithRetry(pipeline, script: "git config --global --add safe.directory \"*\"")
     }
 
+    if (testFilter[(DEBUG_MODE)]) {
+        stage("Interactive debug session")
+        {
+            testFilter[(DEBUG_MODE)] = false
+
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install openssh-server -y")
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install autossh -y")
+            trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install sshpass -y")
+
+            sh """
+                echo 'Port 22' >> /etc/ssh/sshd_config
+                echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config
+                echo 'PasswordAuthentication yes' >> /etc/ssh/sshd_config
+                echo 'PubkeyAuthentication yes' >> /etc/ssh/sshd_config
+                echo 'AllowTcpForwarding yes' >> /etc/ssh/sshd_config
+                echo 'GatewayPorts yes' >> /etc/ssh/sshd_config
+                cat /etc/ssh/sshd_config
+            """
+
+            sh "service ssh restart"
+            sh "service ssh status"
+
+            sh "ssh-keygen -t rsa -b 2048 -f ~/.ssh/id_rsa -N '' -q"
+
+            sh """
+                chmod 700 ~/.ssh
+                chmod 400 ~/.ssh/id_rsa
+                touch ~/.ssh/authorized_keys
+                chmod 600 ~/.ssh/authorized_keys
+            """
+
+            // The portConfig file is in the VM
+            def portConfigFilePath = "/root/.ssh/ports_config.txt"
+
+            withCredentials([
+                usernamePassword(credentialsId: 'tensorrt_llm_infra_debug_vm_01_credentials', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD'),
+                string(credentialsId: 'DEBUG_HOST_NAME', variable: 'HOST_NAME')
+                ]) {
+                sh "sshpass -p ${PASSWORD} -v ssh ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'cat >> ~/.ssh/authorized_keys' < ~/.ssh/id_rsa.pub"
+                sh "ssh -v ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'echo \"\" > ~/.ssh/known_hosts && cat ~/.ssh/id_rsa.pub' >> ~/.ssh/authorized_keys"
+                sh "ssh -v ${USERNAME}@${HOST_NAME} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null 'cat ~/.ssh/ports_config.txt' >> ${portConfigFilePath}"
+
+                def (int userPort, int monitorPort) = getSSHConnectionPorts(portConfigFilePath, stageName)
+                if (userPort == 0) {
+                    echo "Fail to setup an interactive debug session and exit the debug mode."
+                    testFilter[(DEBUG_MODE)] = false
+                    return
+                }
+
+                sh "ssh -f -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -L 1111:127.0.0.1:${monitorPort} -R ${monitorPort}:127.0.0.1:1112 -NR ${userPort}:localhost:22 ${USERNAME}@${HOST_NAME}"
+                sh "autossh -fNR ${userPort}:localhost:22 ${USERNAME}@${HOST_NAME}"
+                sh "ps aux | grep ssh"
+                try {
+                    timeout(time: 2, unit: 'HOURS') {
+                        input message: "Pause 2 hours for Pre-Debug. Please type 'ssh root@${HOST_NAME} -p ${userPort}' on the CLI to create the connection. Please press the button to proceed when you finish debugging."
+                    }
+                } catch (InterruptedException e) {
+                    echo "Pre-debug session was interrupted by user or timeout"
+                    currentBuild.result = 'ABORTED'
+                    error("Pipeline aborted during pre-debug session")
+                } catch (Exception e) {
+                    echo "An error occurred during pre-debug session: ${e.message}"
+                    currentBuild.result = 'FAILURE'
+                    error("Error in pre-debug session: ${e.message}")
+                }
+            }
+
+            testFilter[(DEBUG_MODE)] = true
path: jenkins/L0_Test.groovy
user: login: yiqingy0
body: I set `testFilter[(DEBUG_MODE)]` to `false` at the start of the debug stage https://github.com/NVIDIA/TensorRT-LLM/pull/3091/files#diff-3efa8a6083a90a6f716e4d4bc81710c8104f8bfc5ae642e9fef44df8061e6b7dR748. The `testFilter[(DEBUG_MODE)]` should be false until the debug stage is passed. Otherwise the post-debug will be triggered.
created_at: 2025-04-07T06:17:22Z
author_association: COLLABORATOR

PR comments:
user: login: yiqingy0
created_at: 2025-03-26T06:49:11Z
body: /bot run --stage-list "A10-1"
author_association: COLLABORATOR
user: login: niukuo
created_at: 2025-03-26T06:55:35Z
body: [PR_Github #532](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/532/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: niukuo
created_at: 2025-03-26T08:21:52Z
body: [PR_Github #532](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/532/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #453 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/453/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: yiqingy0
created_at: 2025-03-26T09:47:48Z
body: /bot run --stage-list "A10-1"
author_association: COLLABORATOR
user: login: niukuo
created_at: 2025-03-26T09:54:21Z
body: [PR_Github #564](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/564/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: niukuo
created_at: 2025-03-26T11:24:56Z
body: [PR_Github #564](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/564/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #478 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/478/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: yiqingy0
created_at: 2025-04-07T06:30:33Z
body: /bot run --stage-list "A10-PyTorch-1"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-07T06:36:09Z
body: [PR_Github #1272](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1272/) [ run ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-07T07:16:26Z
body: [PR_Github #1272](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1272/) [ run ] completed with state `SUCCESS`
[/LLM/main/L0_MergeRequest_PR pipeline #958 `(Partly Tested)`](https://nv/trt-llm-cicd/job/main/job/L0_MergeRequest_PR/958/) completed with status: 'SUCCESS'

author_association: COLLABORATOR
user: login: yiqingy0
created_at: 2025-04-07T07:32:07Z
body: /bot skip --comment "add debug stage for the CI pipeline"
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-07T07:37:14Z
body: [PR_Github #1288](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1288/) [ skip ] triggered by Bot
author_association: COLLABORATOR
user: login: tensorrt-cicd
created_at: 2025-04-07T07:46:42Z
body: [PR_Github #1288](https://nv/trt-llm-cicd/job/helpers/job/PR_Github/1288/) [ skip ] completed with state `SUCCESS`
Skipping testing for commit `0d77677`

author_association: COLLABORATOR