Improve CI stability

Signed-off-by: Yanchao Lu <[email protected]>

Author: chzblych

jenkins/L0_Test.groovy

@@ -1,4 +1,4 @@
-@Library(['bloom-jenkins-shared-lib@dev-yanchaol-slurm-output', 'trtllm-jenkins-shared-lib@main']) _
+@Library(['bloom-jenkins-shared-lib@main', 'trtllm-jenkins-shared-lib@main']) _
 
 import java.lang.InterruptedException
 import groovy.transform.Field
@@ -245,7 +245,7 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p
 
                 Utils.exec(pipeline, script: "sshpass -p '${remote.passwd}' scp -r -p ${COMMON_SSH_OPTIONS} ${jenkinsSetupPath} ${remote.user}@${remote.host}:~/bloom/scripts/${nodeName}-slurm_jenkins_agent_setup.sh",)
 
-                sh(label: "Print slurm_jenkins_agent_setup.sh script", script: "cat ${jenkinsSetupPath}")
+                Utils.exec(pipeline, script: "cat ${jenkinsSetupPath}")
 
                 Utils.exec(
                     pipeline,
@@ -297,8 +297,10 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p
             }
         }
     } finally {
-        cleanUpNodeResources(pipeline, cluster, nodeName)
+        Utils.exec(pipeline, script: "echo Sleeping to allow docker stop; sleep 30")
         CloudManager.destroyNode(nodeName)
+        Utils.exec(pipeline, script: "echo Sleeping to allow node destruction; sleep 30")
+        cleanUpNodeResources(pipeline, cluster, nodeName)
     }
 }
 
@@ -321,13 +323,7 @@ def runLLMTestlistOnSlurm_MultiNodes(pipeline, platform, testList, config=VANILL
     String customSuffix = "${env.BUILD_TAG}-${UUID.randomUUID().toString().replaceAll("-", "").substring(0, 6)}".toLowerCase()
     def jobUID = "${cluster.host}-multi_node_test-${customSuffix}"
 
-    sh(
-        label: "Print env for debugging",
-        script: """
-            env | sort
-            pwd && ls -alh
-        """
-    )
+    Utils.exec(pipeline, script: "env | sort && pwd && ls -alh")
 
     try {
         // Run ssh command to start node in desired cluster via SLURM
@@ -371,7 +367,7 @@ def runLLMTestlistOnSlurm_MultiNodes(pipeline, platform, testList, config=VANILL
                 def scriptRunLocalPath = "${llmSrcLocal}/jenkins/scripts/slurm_run.sh"
                 Utils.exec(pipeline, script: "chmod +x ${scriptRunLocalPath}", returnStdout: true)
                 Utils.exec(pipeline, script: "sshpass -p '${remote.passwd}' scp -r -p ${COMMON_SSH_OPTIONS} ${scriptRunLocalPath} ${remote.user}@${remote.host}:${scriptRunNode}",)
-                sh(label: "Print slurm_run.sh script", script: "cat ${scriptRunLocalPath}")
+                Utils.exec(pipeline, script: "cat ${scriptRunLocalPath}")
 
                 // Upload waives.txt to Frontend node
                 def waivesListLocalPath = "${llmSrcLocal}/tests/integration/test_lists/waives.txt"
@@ -429,7 +425,7 @@ def runLLMTestlistOnSlurm_MultiNodes(pipeline, platform, testList, config=VANILL
                 pipeline.writeFile(file: scriptLaunchDestPath, text: scriptContent)
                 Utils.exec(pipeline, script: "chmod +x ${scriptLaunchDestPath}", returnStdout: true)
                 Utils.exec(pipeline, script: "sshpass -p '${remote.passwd}' scp -r -p ${COMMON_SSH_OPTIONS} ${scriptLaunchDestPath} ${remote.user}@${remote.host}:${scriptLaunch}",)
-                sh(label: "Print slurm_launch.sh script", script: "cat ${scriptLaunchDestPath}")
+                Utils.exec(pipeline, script: "cat ${scriptLaunchDestPath}")
             }
             stage('Run Test') {
                 def scriptLaunch = "${jobWorkspace}/slurm_launch.sh"
@@ -2169,10 +2165,19 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null)
                             libEnv += ["LD_LIBRARY_PATH+nvrtc=/usr/local/lib/python${pyver}/dist-packages/nvidia/cuda_nvrtc/lib"]
                         }
                         echo "###### Check pip install Start ######"
+                        def sleepTime = 35
                         withEnv(libEnv) {
-                            sh "env | sort"
-                            timeout(time: 30, unit: 'MINUTES') {
-                                checkPipInstall(pipeline, "${cpu_arch}/${wheelPath}")
+                            retry(1) {
+                                try {
+                                    sh "env | sort"
+                                    timeout(time: 30, unit: 'MINUTES') {
+                                        sleep(sleepTime * 60)
+                                        sleepTime = 1
+                                        checkPipInstall(pipeline, "${cpu_arch}/${wheelPath}")
+                                    }
+                                } catch (org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout e) {
+                                    error "Timeout occurred, retrying..."
+                                }
                             }
                         }
                         echo "###### Run LLMAPI tests Start ######"