From 321ef8f30030d495d0cff0243da35d55ffa3d1d2 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 28 Apr 2026 17:30:44 +0200 Subject: [PATCH 01/35] SRE-3704 ci: NPE - missing ignore_failure key Fix NullPointerException when allowEmptyArchive receives null from missing ignore_failure key When unitTest() throws inside afterTest(), the results_map stash written by runTest() (which lacks the ignore_failure key) is the only stash available to unitTestPost(). Reading results['ignore_failure'] on a Map with no such key returns null in Groovy. Passing null to archiveArtifacts(allowEmptyArchive: null) causes a NullPointerException in Java reflection (unboxBoolean) because ArtifactArchiver.allowEmptyArchive is a primitive boolean. Root cause fix (unitTest.groovy): - Wrap afterTest() in a try/finally block so that the updated stash containing ignore_failure is always written, even if afterTest() throws. Defensive fix (unitTestPost.groovy): - Replace all results['ignore_failure'] accesses with results.get('ignore_failure', false) to guard against any future code path where the stash is incomplete. --- vars/unitTest.groovy | 35 +++++++++++++++++++++-------------- vars/unitTestPost.groovy | 12 ++++++------ 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 8d10191c4..55ed6ef46 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -209,25 +209,32 @@ Map call(Map config = [:]) { p['testResults'] = stage_info.get('testResults', 'test_results/*.xml') p['with_valgrind'] = with_valgrind p['NLT'] = stage_info['NLT'] - runTestData = afterTest(p, runData) - runTestData.each { resultKey, data -> runData[resultKey] = data } + + // Update the stash inside a finally block so that ignore_failure is always + // written even if afterTest() throws. Without this, unitTestPost() reads + // the earlier stash written by runTest() which lacks ignore_failure, causing + // allowEmptyArchive: null and a NullPointerException in ArtifactArchiver. + String results_map = 'results_map_' + sanitizedStageName() + try { + runTestData = afterTest(p, runData) + runTestData.each { resultKey, data -> runData[resultKey] = data } + } finally { + int runTime = durationSeconds(startDate) + runData['unittest_time'] = runTime + + // Use the original ignore_failure setting for post section + runData['ignore_failure'] = config.get('ignore_failure', false) + writeYaml file: results_map, + data: runData, + overwrite: true + stash name: results_map, + includes: results_map + } if (stage_info['compiler'] == 'covc') { stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), includes: 'test.cov' } - int runTime = durationSeconds(startDate) - runData['unittest_time'] = runTime - - // Update the stash after checking junit/valgrind - String results_map = 'results_map_' + sanitizedStageName() - // Use the original ignore_failure setting for post section - runData['ignore_failure'] = config.get('ignore_failure', false) - writeYaml file: results_map, - data: runData, - overwrite: true - stash name: results_map, - includes: results_map // Stash any optional test coverage reports for the stage String code_coverage = 'code_coverage_' + sanitizedStageName() diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index aa081e008..1c30d6326 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -60,7 +60,7 @@ void call(Map config = [:]) { if (testResults != 'None' ) { // groovylint-disable-next-line NoDouble double health_scale = 1.0 - if (results['ignore_failure']) { + if (results.get('ignore_failure', false)) { health_scale = 0.0 } junit testResults: testResults, @@ -81,7 +81,7 @@ void call(Map config = [:]) { class: 'Valgrind', message: 'Valgrind Memcheck error detected', testdata: testdata, - ignoreFailure: results['ignore_failure'] + ignoreFailure: results.get('ignore_failure', false) String memcheck = sanitizedStageName() + '_memcheck_results.tar.bz2' artifact_list.add(memcheck) } @@ -90,12 +90,12 @@ void call(Map config = [:]) { flow_name: flow_name, result: results['result'], junit_files: testResults, - ignore_failure: results['ignore_failure'] + ignore_failure: results.get('ignore_failure', false) artifact_list.each { artifactPat -> println("Archiving Artifacts matching ${artifactPat}") archiveArtifacts artifacts: artifactPat, - allowEmptyArchive: results['ignore_failure'] + allowEmptyArchive: results.get('ignore_failure', false) } String target_stash = "${stage_info['target']}-${stage_info['compiler']}" if (stage_info['build_type']) { @@ -142,9 +142,9 @@ void call(Map config = [:]) { } println results['result_code'].getClass() - println results['ignore_failure'].getClass() + println results.get('ignore_failure', false).getClass() - if (results['result_code'] != 0 && !results['ignore_failure']) { + if (results['result_code'] != 0 && !results.get('ignore_failure', false)) { // Extra information for when this happens. println("results: ${results}") println results['result_code'].getClass() From 29edd4ae23f2cca978b6cba8ac5c9d60df8e69f4 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 28 Apr 2026 21:39:48 +0200 Subject: [PATCH 02/35] Fix duplicate VM_test recordIssues ID across NLT stages Both 'NLT' and 'NLT Fault injection testing' stages have their stage_info['NLT'] set to true by parseStageInfo() because both stage names contain the string 'NLT'. As a result unitTestPost() calls recordIssues with the hardcoded id: 'VM_test' for both stages in the same build, causing: IllegalStateException: ID VM_test is already used by another action Fix by replacing the hardcoded 'VM_test' with sanitizedStageName() + '_VM_test', which produces a unique ID per stage (e.g. 'NLT_VM_test' and 'NLT_Fault_injection_testing_VM_test'). --- vars/unitTestPost.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 1c30d6326..3b0ea0bfc 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -128,7 +128,7 @@ void call(Map config = [:]) { name: 'Node local testing', tool: issues(pattern: 'vm_test/nlt-errors.json', name: 'NLT results', - id: 'VM_test'), + id: sanitizedStageName() + '_VM_test'), scm: 'daos-stack/daos' if (cb_result != currentBuild.result) { From e797b06d55ac6ab31f4b0556b096f64642d7a929 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 29 Apr 2026 12:32:07 +0200 Subject: [PATCH 03/35] Skip memcheck tarball when no memcheck files were produced When a stage runs with memcheck disabled (e.g. NLT Fault injection testing uses '--memcheck no'), no *memcheck.xml files are created. The fileOperations copy step copies 0 files so the target directory is never created, and the unconditional tar command fails with: tar: : Cannot stat: No such file or directory Guard the tar with fileExists() so it is only executed when the memcheck directory was actually populated. Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test-memcheck: true Skip-func-vm-all: true Skip-test-el-9-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: true --- vars/unitTest.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 55ed6ef46..3ac57b9d2 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -106,8 +106,10 @@ Map afterTest(Map config, Map testRunInfo) { flattenFiles: false, includes: valgrind_pattern, targetLocation: memcheck_dir)]) - sh label: 'Create tarball of Valgrind xml files', - script: "tar -cjf ${memcheck_dir}.tar.bz2 ${memcheck_dir}" + if (fileExists(memcheck_dir)) { + sh label: 'Create tarball of Valgrind xml files', + script: "tar -cjf ${memcheck_dir}.tar.bz2 ${memcheck_dir}" + } } if (config['ignore_failure'] && (result['result'] != 'SUCCESS')) { From 280fe8eeb050b3179fcfc1d5018b661bc1dba430 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 29 Apr 2026 12:43:19 +0200 Subject: [PATCH 04/35] Make NLT recordIssues section name configurable in unitTestPost Add a new config['nlt_name'] parameter to unitTestPost() to allow callers to override the display name used for the NLT recordIssues section in the Jenkins UI. Defaults to 'Node local testing' to keep existing behaviour for the NLT stage. The NLT Fault injection testing stage passes nlt_name: 'Fault injection issues' so its warnings section is clearly distinguished from the plain NLT stage in the Jenkins UI. Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test-memcheck: true Skip-func-vm-all: true Skip-test-el-9-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: true --- vars/unitTestPost.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 3b0ea0bfc..8dab0aec3 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -22,9 +22,10 @@ * config['valgrind_stash'] Name to stash Valgrind artifacts * Required if more than one stage is * creating Valgrind reports. - */ - -// groovylint-disable DuplicateStringLiteral, VariableName + * + * config['nlt_name'] Display name for the NLT recordIssues + * section in the Jenkins UI. + * Default: 'Node local testing' /* groovylint-disable-next-line MethodSize */ void call(Map config = [:]) { Map stage_info = parseStageInfo(config) @@ -125,7 +126,7 @@ void call(Map config = [:]) { [threshold: 1, type: 'TOTAL_HIGH'], [threshold: 1, type: 'NEW_NORMAL', unstable: true], [threshold: 1, type: 'NEW_LOW', unstable: true]], - name: 'Node local testing', + name: config.get('nlt_name', 'Node local testing'), tool: issues(pattern: 'vm_test/nlt-errors.json', name: 'NLT results', id: sanitizedStageName() + '_VM_test'), From 503106fa4a3ea6ac2affebb9250d4086d4953f8c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 29 Apr 2026 21:07:47 +0200 Subject: [PATCH 05/35] ci: allow empty memcheck archive in unitTestPost The memcheck tarball (${stage}_memcheck_results.tar.bz2) is only created by unitTest.groovy when memcheck files actually exist (guarded by fileExists(memcheck_dir)). However, unitTestPost.groovy added the tarball to artifact_list unconditionally, and artifact_list is archived with allowEmptyArchive tied to ignore_failure. For the NLT Fault injection testing stage ignore_failure=false, so archiveArtifacts throws: No artifacts found that match the file pattern "NLT_Fault_injection_testing_memcheck_results.tar.bz2". Configuration error? Archive the memcheck tarball directly with allowEmptyArchive: true instead of adding it to artifact_list, so it is silently skipped when no memcheck files were produced. Signed-off-by: Tomasz Gromadzki Priority: 2 Cancel-prev-build: false Skip-python-bandit: true Skip-unit-test-memcheck: true Skip-func-vm-all: true Skip-test-el-9-rpms: true Skip-test-leap-15-rpms: true Skip-func-hw-test: true Skip-build-el8-gcc: true Skip-build-leap15-gcc: true --- vars/unitTestPost.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 8dab0aec3..674655074 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -84,7 +84,7 @@ void call(Map config = [:]) { testdata: testdata, ignoreFailure: results.get('ignore_failure', false) String memcheck = sanitizedStageName() + '_memcheck_results.tar.bz2' - artifact_list.add(memcheck) + archiveArtifacts artifacts: memcheck, allowEmptyArchive: true } stepResult name: description, context: context, From 7267d71e2673fbc89c9926a40e1e7f5252b302b3 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 30 Apr 2026 10:59:10 +0200 Subject: [PATCH 06/35] NLT: distinguish fault injection stage from regular NLT parseStageInfo: detect 'NLT fault injection' stage separately and set FI=true in addition to NLT=true, leaving the regular NLT path unchanged (with valgrind enabled). unitTest/unitTestPost: remove NLT from the valgrind check condition since fault injection runs with --memcheck no and produces no memcheck files. unitTestPost: when FI=true, add nlt-client-leaks.json as a second tool to the recordIssues call alongside the existing vm_test/nlt-errors.json. skipStage: add 'NLT Fault injection testing' case to match the stage name used in the Jenkinsfile. Signed-off-by: Tomasz Gromadzki --- vars/parseStageInfo.groovy | 7 ++++++- vars/skipStage.groovy | 1 + vars/unitTest.groovy | 3 +-- vars/unitTestPost.groovy | 22 ++++++++++++++-------- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/vars/parseStageInfo.groovy b/vars/parseStageInfo.groovy index bf7719c60..7761bddcf 100755 --- a/vars/parseStageInfo.groovy +++ b/vars/parseStageInfo.groovy @@ -285,7 +285,12 @@ Map call(Map config = [:]) { result['ftest_arg'] = config['ftest_arg'] } - if (stage_name.contains('NLT')) { + if (stage_name.toLowerCase().contains('nlt fault injection')) { + result['NLT'] = true + result['FI'] = true + result['always_script'] = config.get('always_script', 'ci/unit/test_nlt_post.sh') + result['testResults'] = config.get('testResults', 'nlt-junit.xml') + } else if (stage_name.contains('NLT')) { result['NLT'] = true result['valgrind_pattern'] = config.get('valgrind_pattern', '*memcheck.xml') result['always_script'] = config.get('always_script', 'ci/unit/test_nlt_post.sh') diff --git a/vars/skipStage.groovy b/vars/skipStage.groovy index 17ec7d388..61f2c4440 100644 --- a/vars/skipStage.groovy +++ b/vars/skipStage.groovy @@ -434,6 +434,7 @@ boolean call(Map config = [:]) { skip_ftest('ubuntu20', target_branch, tags) */ return true case 'Fault injection testing': + case 'NLT Fault injection testing': case 'Fault injection testing on CentOS 8': case 'Fault injection testing on EL 8': case 'Fault injection testing on EL 8.8': diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 3ac57b9d2..0c2d8efaf 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -94,7 +94,7 @@ Map afterTest(Map config, Map testRunInfo) { } else { result['result'] = checkJunitFiles(testResults: testResults) } - if (config['with_valgrind'] || config['NLT']) { + if (config['with_valgrind']) { vgrcs = sh label: 'Check for Valgrind errors', script: "grep -E ')' ${valgrind_pattern} || true", returnStdout: true @@ -210,7 +210,6 @@ Map call(Map config = [:]) { 'unit-test-*memcheck.xml') p['testResults'] = stage_info.get('testResults', 'test_results/*.xml') p['with_valgrind'] = with_valgrind - p['NLT'] = stage_info['NLT'] // Update the stash inside a finally block so that ignore_failure is always // written even if afterTest() throws. Without this, unitTestPost() reads diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 674655074..1fc405923 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -22,10 +22,10 @@ * config['valgrind_stash'] Name to stash Valgrind artifacts * Required if more than one stage is * creating Valgrind reports. - * - * config['nlt_name'] Display name for the NLT recordIssues - * section in the Jenkins UI. - * Default: 'Node local testing' + * + * config['nlt_name'] Display name for the NLT recordIssues + * section in the Jenkins UI. + * Default: 'Node local testing' /* groovylint-disable-next-line MethodSize */ void call(Map config = [:]) { Map stage_info = parseStageInfo(config) @@ -67,7 +67,7 @@ void call(Map config = [:]) { junit testResults: testResults, healthScaleFactor: health_scale } - if (stage_info['with_valgrind'] || stage_info['NLT']) { + if (stage_info['with_valgrind']) { String suite = sanitizedStageName() int vgfail = 0 String testdata @@ -113,6 +113,14 @@ void call(Map config = [:]) { 'daos-stack/daos/master'), scm: 'daos-stack/daos', requiredResult: 'UNSTABLE') + List nltTools = [issues(pattern: 'vm_test/nlt-errors.json', + name: 'NLT results', + id: sanitizedStageName() + '_VM_test')] + if (stage_info['FI']) { + nltTools << issues(pattern: 'nlt-client-leaks.json', + name: 'Fault injection leaks', + id: 'NLT_client') + } recordIssues enabledForFailure: true, /* ignore warning/errors from PMDK logging system */ filters: [excludeFile('pmdk/.+')], @@ -127,9 +135,7 @@ void call(Map config = [:]) { [threshold: 1, type: 'NEW_NORMAL', unstable: true], [threshold: 1, type: 'NEW_LOW', unstable: true]], name: config.get('nlt_name', 'Node local testing'), - tool: issues(pattern: 'vm_test/nlt-errors.json', - name: 'NLT results', - id: sanitizedStageName() + '_VM_test'), + tools: nltTools, scm: 'daos-stack/daos' if (cb_result != currentBuild.result) { From be5702cf2dd2fd9061833e3caadc1577e63abd32 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 5 May 2026 15:47:44 +0200 Subject: [PATCH 07/35] Normalize names on Jenkis GUI Signed-off-by: Tomasz Gromadzki --- vars/unitTestPost.groovy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 1fc405923..aff9bcab3 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -25,7 +25,7 @@ * * config['nlt_name'] Display name for the NLT recordIssues * section in the Jenkins UI. - * Default: 'Node local testing' + * Default: 'NLT' /* groovylint-disable-next-line MethodSize */ void call(Map config = [:]) { Map stage_info = parseStageInfo(config) @@ -114,9 +114,12 @@ void call(Map config = [:]) { scm: 'daos-stack/daos', requiredResult: 'UNSTABLE') List nltTools = [issues(pattern: 'vm_test/nlt-errors.json', - name: 'NLT results', + name: 'NLT errors', id: sanitizedStageName() + '_VM_test')] if (stage_info['FI']) { + nltTools = [issues(pattern: 'vm_test/nlt-errors.json', + name: 'Fault injection', + id: sanitizedStageName() + '_VM_test')] nltTools << issues(pattern: 'nlt-client-leaks.json', name: 'Fault injection leaks', id: 'NLT_client') @@ -134,7 +137,7 @@ void call(Map config = [:]) { [threshold: 1, type: 'TOTAL_HIGH'], [threshold: 1, type: 'NEW_NORMAL', unstable: true], [threshold: 1, type: 'NEW_LOW', unstable: true]], - name: config.get('nlt_name', 'Node local testing'), + name: config.get('nlt_name', 'NLT'), tools: nltTools, scm: 'daos-stack/daos' From f3c82436094c6d5548864a8d22185550c5b1d4f1 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 5 May 2026 21:39:28 +0200 Subject: [PATCH 08/35] Small optimization of code Signed-off-by: Tomasz Gromadzki --- vars/parseStageInfo.groovy | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/vars/parseStageInfo.groovy b/vars/parseStageInfo.groovy index 7761bddcf..6d7c1f330 100755 --- a/vars/parseStageInfo.groovy +++ b/vars/parseStageInfo.groovy @@ -285,17 +285,16 @@ Map call(Map config = [:]) { result['ftest_arg'] = config['ftest_arg'] } - if (stage_name.toLowerCase().contains('nlt fault injection')) { + if(stage_name.contains('NLT')) { result['NLT'] = true - result['FI'] = true result['always_script'] = config.get('always_script', 'ci/unit/test_nlt_post.sh') result['testResults'] = config.get('testResults', 'nlt-junit.xml') - } else if (stage_name.contains('NLT')) { - result['NLT'] = true - result['valgrind_pattern'] = config.get('valgrind_pattern', '*memcheck.xml') - result['always_script'] = config.get('always_script', 'ci/unit/test_nlt_post.sh') - result['testResults'] = config.get('testResults', 'nlt-junit.xml') - result['with_valgrind'] = 'memcheck' + if (stage_name.toLowerCase().contains('nlt fault injection')) { + result['FI'] = true + } else { + result['valgrind_pattern'] = config.get('valgrind_pattern', '*memcheck.xml') + result['with_valgrind'] = 'memcheck' + } } else { result['NLT'] = false if (config['valgrind_pattern']) { From 633efd87bfc476951b6b278b908e6acb5d329abd Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 6 May 2026 12:15:32 +0200 Subject: [PATCH 09/35] Remove vm_test/ prefix from path to nlt-errors.json file Signed-off-by: Tomasz Gromadzki --- vars/unitTestPost.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index aff9bcab3..78ad940a5 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -113,11 +113,11 @@ void call(Map config = [:]) { 'daos-stack/daos/master'), scm: 'daos-stack/daos', requiredResult: 'UNSTABLE') - List nltTools = [issues(pattern: 'vm_test/nlt-errors.json', + List nltTools = [issues(pattern: 'nlt-errors.json', name: 'NLT errors', id: sanitizedStageName() + '_VM_test')] if (stage_info['FI']) { - nltTools = [issues(pattern: 'vm_test/nlt-errors.json', + nltTools = [issues(pattern: 'nlt-errors.json', name: 'Fault injection', id: sanitizedStageName() + '_VM_test')] nltTools << issues(pattern: 'nlt-client-leaks.json', From bb73e9957832edc1d730f7d3d78ee92b51b66636 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 6 May 2026 12:16:15 +0200 Subject: [PATCH 10/35] Revert "Remove vm_test/ prefix from path to nlt-errors.json file" This reverts commit df0bd595dd5db27d5b32bbb9bf99da019822fa89. --- vars/unitTestPost.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 78ad940a5..aff9bcab3 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -113,11 +113,11 @@ void call(Map config = [:]) { 'daos-stack/daos/master'), scm: 'daos-stack/daos', requiredResult: 'UNSTABLE') - List nltTools = [issues(pattern: 'nlt-errors.json', + List nltTools = [issues(pattern: 'vm_test/nlt-errors.json', name: 'NLT errors', id: sanitizedStageName() + '_VM_test')] if (stage_info['FI']) { - nltTools = [issues(pattern: 'nlt-errors.json', + nltTools = [issues(pattern: 'vm_test/nlt-errors.json', name: 'Fault injection', id: sanitizedStageName() + '_VM_test')] nltTools << issues(pattern: 'nlt-client-leaks.json', From 8e3296c70799e1200b995e164aeea0abb54af307 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 8 May 2026 07:14:40 +0200 Subject: [PATCH 11/35] Ensure 20 cores per VM for NLT tests Signed-off-by: Tomasz Gromadzki --- vars/unitTest.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 0c2d8efaf..66f974d2b 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -148,7 +148,8 @@ Map call(Map config = [:]) { node_count: stage_info['node_count'], distro: image_version, inst_repos: config.get('inst_repos', ''), - inst_rpms: inst_rpms) + inst_rpms: inst_rpms, + environment: stage_info['NLT']?"VM_CPUS=20":"") /* el9-gcc-tests */ String target_stash = (image_version ?: ${stage_info['target']}).split('\\.')[0] From 2cf4488f34ead4244a762627968a18b4b1f83c9b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 8 May 2026 12:39:10 +0200 Subject: [PATCH 12/35] config['nlt_name'] no longer needed Signed-off-by: Tomasz Gromadzki --- vars/unitTestPost.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index aff9bcab3..771fdc0b3 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -23,9 +23,6 @@ * Required if more than one stage is * creating Valgrind reports. * - * config['nlt_name'] Display name for the NLT recordIssues - * section in the Jenkins UI. - * Default: 'NLT' /* groovylint-disable-next-line MethodSize */ void call(Map config = [:]) { Map stage_info = parseStageInfo(config) @@ -137,7 +134,7 @@ void call(Map config = [:]) { [threshold: 1, type: 'TOTAL_HIGH'], [threshold: 1, type: 'NEW_NORMAL', unstable: true], [threshold: 1, type: 'NEW_LOW', unstable: true]], - name: config.get('nlt_name', 'NLT'), + name: stage_info['FI']?'Fault injection':'NLT', tools: nltTools, scm: 'daos-stack/daos' From 6fe3ebed7a98b45f4f05def7551567953cea07d7 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 11 May 2026 08:37:18 +0200 Subject: [PATCH 13/35] Fix: typo Doc-only: true Signed-off-by: Tomasz Gromadzki --- vars/distroVersion.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/vars/distroVersion.groovy b/vars/distroVersion.groovy index 29d58fa2f..426da66d4 100755 --- a/vars/distroVersion.groovy +++ b/vars/distroVersion.groovy @@ -30,6 +30,7 @@ String call(String distro) { } String call(String distro, String branch) { + println("Determining distro version for ${distro} and branch ${branch}") return ['el8': ['master': '8.8', '2.4': '8.8', '2.6': '8.8', From 62ad22190f4ece19540c92d7147fea346145fddc Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 11 May 2026 08:58:26 +0200 Subject: [PATCH 14/35] Fix: sles15 is missing Doc-only: true Signed-off-by: Tomasz Gromadzki --- vars/rpmDistValue.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/rpmDistValue.groovy b/vars/rpmDistValue.groovy index 6839bf8af..01599c948 100644 --- a/vars/rpmDistValue.groovy +++ b/vars/rpmDistValue.groovy @@ -21,7 +21,7 @@ String call(String distro) { distro.startsWith('rocky9') || distro.startsWith('almalinux9') || distro.startsWith('rhel9')) { return '.el9' - } else if (distro.startsWith('leap15')) { + } else if (distro.startsWith('leap15') || distro.startsWith('sles15')) { return '.suse.lp' + parseStageInfo()['distro_version'].replaceAll('\\.', '') } error("Don't know what the RPM %{dist} is for ${distro}") From f4942bfcbdae5f8cf71c390c0f097d33d2218b14 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 11 May 2026 09:04:32 +0200 Subject: [PATCH 15/35] Stable version ready for review Signed-off-by: Tomasz Gromadzki --- vars/distroVersion.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/vars/distroVersion.groovy b/vars/distroVersion.groovy index 426da66d4..62fd711a0 100755 --- a/vars/distroVersion.groovy +++ b/vars/distroVersion.groovy @@ -30,7 +30,6 @@ String call(String distro) { } String call(String distro, String branch) { - println("Determining distro version for ${distro} and branch ${branch}") return ['el8': ['master': '8.8', '2.4': '8.8', '2.6': '8.8', @@ -43,7 +42,6 @@ String call(String distro, String branch) { '2.6': '15.6', '2.8': '15.6'], 'sles15': ['master': '15.7', - '2.4': '15.7', '2.6': '15.7', '2.8': '15.7'], 'ubuntu20': ['master': '20.04']][distro][branch] From 9ceceff9ca51bda45b53787c70853da3f6f68135 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 11 May 2026 22:47:46 +0200 Subject: [PATCH 16/35] Test with 14 cores Doc-only: true Signed-off-by: Tomasz Gromadzki --- vars/unitTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 66f974d2b..cb11b0f54 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -149,7 +149,7 @@ Map call(Map config = [:]) { distro: image_version, inst_repos: config.get('inst_repos', ''), inst_rpms: inst_rpms, - environment: stage_info['NLT']?"VM_CPUS=20":"") + environment: stage_info['NLT']?"VM_CPUS=14":"") /* el9-gcc-tests */ String target_stash = (image_version ?: ${stage_info['target']}).split('\\.')[0] From c64934d4bcf82c723a516c2c70b1c0931e1c085f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 12 May 2026 14:04:43 +0200 Subject: [PATCH 17/35] Fix: reply to review Signed-off-by: Tomasz Gromadzki --- vars/distroVersion.groovy | 1 + vars/unitTest.groovy | 2 +- vars/unitTestPost.groovy | 5 +---- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/vars/distroVersion.groovy b/vars/distroVersion.groovy index 62fd711a0..29d58fa2f 100755 --- a/vars/distroVersion.groovy +++ b/vars/distroVersion.groovy @@ -42,6 +42,7 @@ String call(String distro, String branch) { '2.6': '15.6', '2.8': '15.6'], 'sles15': ['master': '15.7', + '2.4': '15.7', '2.6': '15.7', '2.8': '15.7'], 'ubuntu20': ['master': '20.04']][distro][branch] diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index cb11b0f54..519bfe971 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -149,7 +149,7 @@ Map call(Map config = [:]) { distro: image_version, inst_repos: config.get('inst_repos', ''), inst_rpms: inst_rpms, - environment: stage_info['NLT']?"VM_CPUS=14":"") + prov_env_vars: stage_info['NLT']?'VM_CPUS=14':'') /* el9-gcc-tests */ String target_stash = (image_version ?: ${stage_info['target']}).split('\\.')[0] diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 771fdc0b3..fddd548e6 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -111,12 +111,9 @@ void call(Map config = [:]) { scm: 'daos-stack/daos', requiredResult: 'UNSTABLE') List nltTools = [issues(pattern: 'vm_test/nlt-errors.json', - name: 'NLT errors', + name: stage_info['FI'] ? 'Fault injection' : 'NLT errors', id: sanitizedStageName() + '_VM_test')] if (stage_info['FI']) { - nltTools = [issues(pattern: 'vm_test/nlt-errors.json', - name: 'Fault injection', - id: sanitizedStageName() + '_VM_test')] nltTools << issues(pattern: 'nlt-client-leaks.json', name: 'Fault injection leaks', id: 'NLT_client') From 4a52cb2a042586222dd4e32d1e58ecadbc19b271 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 14 May 2026 09:52:40 +0200 Subject: [PATCH 18/35] unitTest with prov_env_vars parameter Signed-off-by: Tomasz Gromadzki --- vars/unitTest.groovy | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 519bfe971..cf48bc6a1 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -72,7 +72,13 @@ * * config['unstash_tests'] Un-stash -tests, default is true. * - * config['image_version'] Image version to use for provisioning, e.g. el8.8, leap15.6, etc. + * config['image_version'] Image version to use for provisioning, + * e.g. el8.8, leap15.6, etc. + * + * config['prov_env_vars'] Optional provisioning environment to use. + * Default ''. + * Formatted as 'KEY=VALUE' space-separated pairs + * and passed to the provisionNodesSystem call. */ Map afterTest(Map config, Map testRunInfo) { @@ -149,7 +155,7 @@ Map call(Map config = [:]) { distro: image_version, inst_repos: config.get('inst_repos', ''), inst_rpms: inst_rpms, - prov_env_vars: stage_info['NLT']?'VM_CPUS=14':'') + prov_env_vars: config.get('prov_env_vars', '')) /* el9-gcc-tests */ String target_stash = (image_version ?: ${stage_info['target']}).split('\\.')[0] From a068dc849e4a780765494c2b7ead7b181a91e645 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 14 May 2026 09:58:35 +0200 Subject: [PATCH 19/35] Revert debug code: fileExists(memcheck_dir) Signed-off-by: Tomasz Gromadzki --- vars/unitTest.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index cf48bc6a1..c9a0f6117 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -112,10 +112,8 @@ Map afterTest(Map config, Map testRunInfo) { flattenFiles: false, includes: valgrind_pattern, targetLocation: memcheck_dir)]) - if (fileExists(memcheck_dir)) { - sh label: 'Create tarball of Valgrind xml files', - script: "tar -cjf ${memcheck_dir}.tar.bz2 ${memcheck_dir}" - } + sh label: 'Create tarball of Valgrind xml files', + script: "tar -cjf ${memcheck_dir}.tar.bz2 ${memcheck_dir}" } if (config['ignore_failure'] && (result['result'] != 'SUCCESS')) { From 4d4107ec217c17a19eada6462c883bc3bcfef7d8 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 14 May 2026 10:04:54 +0200 Subject: [PATCH 20/35] Revert debug code: results.get('ignore_failure', false) Signed-off-by: Tomasz Gromadzki --- vars/unitTestPost.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index fddd548e6..43495a1ea 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -22,7 +22,7 @@ * config['valgrind_stash'] Name to stash Valgrind artifacts * Required if more than one stage is * creating Valgrind reports. - * + */ /* groovylint-disable-next-line MethodSize */ void call(Map config = [:]) { Map stage_info = parseStageInfo(config) @@ -58,7 +58,7 @@ void call(Map config = [:]) { if (testResults != 'None' ) { // groovylint-disable-next-line NoDouble double health_scale = 1.0 - if (results.get('ignore_failure', false)) { + if (results['ignore_failure']) { health_scale = 0.0 } junit testResults: testResults, @@ -79,7 +79,7 @@ void call(Map config = [:]) { class: 'Valgrind', message: 'Valgrind Memcheck error detected', testdata: testdata, - ignoreFailure: results.get('ignore_failure', false) + ignoreFailure: results['ignore_failure'] String memcheck = sanitizedStageName() + '_memcheck_results.tar.bz2' archiveArtifacts artifacts: memcheck, allowEmptyArchive: true } @@ -88,12 +88,12 @@ void call(Map config = [:]) { flow_name: flow_name, result: results['result'], junit_files: testResults, - ignore_failure: results.get('ignore_failure', false) + ignore_failure: results['ignore_failure'] artifact_list.each { artifactPat -> println("Archiving Artifacts matching ${artifactPat}") archiveArtifacts artifacts: artifactPat, - allowEmptyArchive: results.get('ignore_failure', false) + allowEmptyArchive: results['ignore_failure'] } String target_stash = "${stage_info['target']}-${stage_info['compiler']}" if (stage_info['build_type']) { @@ -146,9 +146,9 @@ void call(Map config = [:]) { } println results['result_code'].getClass() - println results.get('ignore_failure', false).getClass() + println results['ignore_failure'].getClass() - if (results['result_code'] != 0 && !results.get('ignore_failure', false)) { + if (results['result_code'] != 0 && !results['ignore_failure']) { // Extra information for when this happens. println("results: ${results}") println results['result_code'].getClass() From 4a4efea4acfd2d993f9b9279793c430cc42fab27 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 14 May 2026 11:19:02 +0200 Subject: [PATCH 21/35] Revert debug code: artifact_list.add(memcheck) Signed-off-by: Tomasz Gromadzki --- vars/unitTestPost.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 43495a1ea..72f85e1d6 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -81,7 +81,7 @@ void call(Map config = [:]) { testdata: testdata, ignoreFailure: results['ignore_failure'] String memcheck = sanitizedStageName() + '_memcheck_results.tar.bz2' - archiveArtifacts artifacts: memcheck, allowEmptyArchive: true + artifact_list.add(memcheck) } stepResult name: description, context: context, From fc92b8b74b9c344e488bec0997e828960bf49cc6 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 14 May 2026 11:22:34 +0200 Subject: [PATCH 22/35] Revert debug code: unitTest try - final Signed-off-by: Tomasz Gromadzki --- vars/unitTest.groovy | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index c9a0f6117..8ca9be861 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -215,33 +215,25 @@ Map call(Map config = [:]) { 'unit-test-*memcheck.xml') p['testResults'] = stage_info.get('testResults', 'test_results/*.xml') p['with_valgrind'] = with_valgrind - - // Update the stash inside a finally block so that ignore_failure is always - // written even if afterTest() throws. Without this, unitTestPost() reads - // the earlier stash written by runTest() which lacks ignore_failure, causing - // allowEmptyArchive: null and a NullPointerException in ArtifactArchiver. - String results_map = 'results_map_' + sanitizedStageName() - try { - runTestData = afterTest(p, runData) - runTestData.each { resultKey, data -> runData[resultKey] = data } - } finally { - int runTime = durationSeconds(startDate) - runData['unittest_time'] = runTime - - // Use the original ignore_failure setting for post section - runData['ignore_failure'] = config.get('ignore_failure', false) - writeYaml file: results_map, - data: runData, - overwrite: true - stash name: results_map, - includes: results_map - } - + p['NLT'] = stage_info['NLT'] + runTestData = afterTest(p, runData) + runTestData.each { resultKey, data -> runData[resultKey] = data } + if (stage_info['compiler'] == 'covc') { stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), includes: 'test.cov' } + // Update the stash after checking junit/valgrind + String results_map = 'results_map_' + sanitizedStageName() + // Use the original ignore_failure setting for post section + runData['ignore_failure'] = config.get('ignore_failure', false) + writeYaml file: results_map, + data: runData, + overwrite: true + stash name: results_map, + includes: results_map + // Stash any optional test coverage reports for the stage String code_coverage = 'code_coverage_' + sanitizedStageName() stash name: code_coverage, From b172622a98cdb65cf01bd11a93ca2dde47fde37b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 14 May 2026 11:25:55 +0200 Subject: [PATCH 23/35] Fix: Remove spaces Signed-off-by: Tomasz Gromadzki --- vars/unitTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 8ca9be861..7ab2ed5aa 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -113,7 +113,7 @@ Map afterTest(Map config, Map testRunInfo) { includes: valgrind_pattern, targetLocation: memcheck_dir)]) sh label: 'Create tarball of Valgrind xml files', - script: "tar -cjf ${memcheck_dir}.tar.bz2 ${memcheck_dir}" + script: "tar -cjf ${memcheck_dir}.tar.bz2 ${memcheck_dir}" } if (config['ignore_failure'] && (result['result'] != 'SUCCESS')) { @@ -218,7 +218,7 @@ Map call(Map config = [:]) { p['NLT'] = stage_info['NLT'] runTestData = afterTest(p, runData) runTestData.each { resultKey, data -> runData[resultKey] = data } - + if (stage_info['compiler'] == 'covc') { stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), includes: 'test.cov' From a2889d35bc7e219b5906eb3f10f0b85b1429f79f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 14 May 2026 11:28:29 +0200 Subject: [PATCH 24/35] Revert debug code: unitTest try - final 2nd Signed-off-by: Tomasz Gromadzki --- vars/unitTest.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 7ab2ed5aa..2d428f8d5 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -223,7 +223,9 @@ Map call(Map config = [:]) { stash name: config.get('coverage_stash', "${target_stash}-unit-cov"), includes: 'test.cov' } - + int runTime = durationSeconds(startDate) + runData['unittest_time'] = runTime + // Update the stash after checking junit/valgrind String results_map = 'results_map_' + sanitizedStageName() // Use the original ignore_failure setting for post section From 336fbca949f9b8b46d7de1dc0781f9bf5c1867cd Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 14 May 2026 11:29:01 +0200 Subject: [PATCH 25/35] Revert debug code: unitTest try - final 3rd Signed-off-by: Tomasz Gromadzki --- vars/unitTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 2d428f8d5..6a740ac6f 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -225,7 +225,7 @@ Map call(Map config = [:]) { } int runTime = durationSeconds(startDate) runData['unittest_time'] = runTime - + // Update the stash after checking junit/valgrind String results_map = 'results_map_' + sanitizedStageName() // Use the original ignore_failure setting for post section From 5ee695ded50fdc77be6044aaf647663f99c700da Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 14 May 2026 13:23:22 +0200 Subject: [PATCH 26/35] Linting: spaces in wrong places Signed-off-by: Tomasz Gromadzki --- vars/parseStageInfo.groovy | 2 +- vars/unitTest.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vars/parseStageInfo.groovy b/vars/parseStageInfo.groovy index 6d7c1f330..a0aeb8bf0 100755 --- a/vars/parseStageInfo.groovy +++ b/vars/parseStageInfo.groovy @@ -285,7 +285,7 @@ Map call(Map config = [:]) { result['ftest_arg'] = config['ftest_arg'] } - if(stage_name.contains('NLT')) { + if (stage_name.contains('NLT')) { result['NLT'] = true result['always_script'] = config.get('always_script', 'ci/unit/test_nlt_post.sh') result['testResults'] = config.get('testResults', 'nlt-junit.xml') diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 6a740ac6f..4f002d6ea 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -225,7 +225,7 @@ Map call(Map config = [:]) { } int runTime = durationSeconds(startDate) runData['unittest_time'] = runTime - + // Update the stash after checking junit/valgrind String results_map = 'results_map_' + sanitizedStageName() // Use the original ignore_failure setting for post section From 3125dc90133a7ff5539cabf8f677fa9bcdf08213 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 15 May 2026 09:23:41 +0200 Subject: [PATCH 27/35] Skip NLT/FI tests when the required el-gcc build is skipped NLT and Fault injection testing both consume el-gcc build artifacts (install tree and .build_vars.*). If the corresponding 'Build on EL N' stage is skipped, attempting to run these test stages will fail because the required stashes are never created. Add skip_build_on_el_gcc(target_branch, ) to the 'NLT' and 'Fault injection testing' cases so they are skipped under the same conditions as the build stage that produces their inputs: - Skip-build-elN-gcc: true pragma - doc-only changes with no prRepos for elN - quickFunctional() builds The EL major version is derived dynamically from parseStageInfo(), which is now called once and captured as stageInfo rather than called inline only for test_tag. The version is extracted inline via: stageInfo['target'].replaceFirst('^el(\\d+).*', '$1') This handles all versioned stage names (e.g. 'NLT on EL 8.8' -> '8', 'Fault injection testing on EL 8' -> '8'). For 'NLT Fault injection testing', which has no EL version in its name, parseStageInfo() falls back to 'el9', so the check correctly guards the el9-gcc build. All 'Fault injection testing' cases, including 'NLT Fault injection testing', are unified into a single switch block. This mirrors the existing behaviour of the 'Unit Tests' case. Also remove the now-redundant standalone quickFunctional() and docOnlyChange(target_branch) conditions from the 'Fault injection testing' case -- both are already subsumed by skip_build_on_el_gcc(). Signed-off-by: Tomasz Gromadzki --- vars/skipStage.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vars/skipStage.groovy b/vars/skipStage.groovy index 61f2c4440..c875a293b 100644 --- a/vars/skipStage.groovy +++ b/vars/skipStage.groovy @@ -158,7 +158,8 @@ boolean call(Map config = [:]) { } String target_branch = env.CHANGE_TARGET ? env.CHANGE_TARGET : env.BRANCH_NAME - String tags = config['tags'] ?: parseStageInfo()['test_tag'] + Map stageInfo = parseStageInfo() + String tags = config['tags'] ?: stageInfo['test_tag'] switch (env.STAGE_NAME) { case 'Cancel Previous Builds': @@ -353,6 +354,7 @@ boolean call(Map config = [:]) { case 'NLT on EL 8': case 'NLT on EL 8.8': return skip_stage_pragma('nlt') || + skip_build_on_el_gcc(target_branch, stageInfo['target'].replaceFirst('^el(\\d+).*', '$1')) || quickBuild() || stageAlreadyPassed() case 'Unit Test Bullseye': @@ -441,10 +443,8 @@ boolean call(Map config = [:]) { return skip_stage_pragma('fault-injection-test') || !paramsValue('CI_FI_TEST', true) || !paramsValue('CI_FI_el8_TEST', true) || /* release/2.6 Jenkinsfile still uses CI_FI_el8_TEST */ - quickFunctional() || - docOnlyChange(target_branch) || + skip_build_on_el_gcc(target_branch, stageInfo['target'].replaceFirst('^el(\\d+).*', '$1')) || skip_stage_pragma('func-test') || - skip_stage_pragma('func-test-vm') || stageAlreadyPassed() case 'Test CentOS 7 RPMs': return !paramsValue('CI_RPMS_el7_TEST', true) || From fd71e7057a18935b146033375a8362ecc60ea7a4 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 15 May 2026 13:17:21 +0200 Subject: [PATCH 28/35] unitTest/Post: allow config to override stage_info defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several parameters (always_script, valgrind_pattern, testResults, with_valgrind) were read exclusively from stage_info, making it impossible for callers to override them without changing parseStageInfo. Apply the standard config -> stage_info -> hardcoded-default priority chain to all of them in both unitTest.groovy and unitTestPost.groovy. parseStageInfo.groovy: remove FI auto-detection from stage name The 'nlt fault injection' stage name check that set result['FI'] = true and suppressed valgrind for FI stages is removed. FI is now indicated solely by config['FI'] passed by the caller. As a consequence, valgrind_pattern and with_valgrind are now set unconditionally for all NLT stages (including Fault injection testing), which is the correct behaviour since FI testing also runs under valgrind. unitTestPost.groovy: replace stage_info['NLT'/'FI'] with config lookups - fi: config.get('FI', false) — no stage_info fallback since FI is no longer auto-detected - nlt: fi || config.get('NLT', stage_info.get('NLT', false)) — keeps auto-detection via stage name for plain NLT stages while ensuring FI always implies NLT - testResults now respects config['testResults'] override - valgrind_pattern lookup scoped to the valgrind_stash block where it is actually used unitTest.groovy: remove p['NLT'] assignment afterTest() never used it and unitTestPost resolves NLT independently from its own config map. Document the new overridable parameters in the header comment. Signed-off-by: Tomasz Gromadzki --- vars/parseStageInfo.groovy | 8 ++------ vars/unitTest.groovy | 33 ++++++++++++++++++++++++++------- vars/unitTestPost.groovy | 27 +++++++++++++++++++-------- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/vars/parseStageInfo.groovy b/vars/parseStageInfo.groovy index a0aeb8bf0..bf7719c60 100755 --- a/vars/parseStageInfo.groovy +++ b/vars/parseStageInfo.groovy @@ -287,14 +287,10 @@ Map call(Map config = [:]) { if (stage_name.contains('NLT')) { result['NLT'] = true + result['valgrind_pattern'] = config.get('valgrind_pattern', '*memcheck.xml') result['always_script'] = config.get('always_script', 'ci/unit/test_nlt_post.sh') result['testResults'] = config.get('testResults', 'nlt-junit.xml') - if (stage_name.toLowerCase().contains('nlt fault injection')) { - result['FI'] = true - } else { - result['valgrind_pattern'] = config.get('valgrind_pattern', '*memcheck.xml') - result['with_valgrind'] = 'memcheck' - } + result['with_valgrind'] = 'memcheck' } else { result['NLT'] = false if (config['valgrind_pattern']) { diff --git a/vars/unitTest.groovy b/vars/unitTest.groovy index 4f002d6ea..1733c0099 100755 --- a/vars/unitTest.groovy +++ b/vars/unitTest.groovy @@ -79,6 +79,21 @@ * Default ''. * Formatted as 'KEY=VALUE' space-separated pairs * and passed to the provisionNodesSystem call. + * + * config['testResults'] Junit test result files. + * Default 'test_results/*.xml' + * + * config['with_valgrind'] Valgrind tools name (e.g. memcheck). + * Default is '' (no Valgrind). + * + * config['always_script'] Script to run for cleanup and artifact collection + * Default is 'ci/unit/test_post_always.sh'. This + * script will be run even if the test script fails + * to allow for artifact collection and cleanup. + * + * config['valgrind_pattern'] Pattern for Valgrind files. + * Default: 'unit-test-*.memcheck.xml' + * */ Map afterTest(Map config, Map testRunInfo) { @@ -187,7 +202,8 @@ Map call(Map config = [:]) { outputFile: 'bullseye.tar' } - String with_valgrind = stage_info.get('with_valgrind', '') + String with_valgrind = config.get('with_valgrind', + stage_info.get('with_valgrind', '')) Map p = [:] p['stashes'] = stashes p['script'] = "SSH_KEY_ARGS=${env.SSH_KEY_ARGS} " + @@ -209,13 +225,16 @@ Map call(Map config = [:]) { runTestData = runTest p runTestData.each { resultKey, data -> runData[resultKey] = data } } - p['always_script'] = stage_info.get('always_script', - 'ci/unit/test_post_always.sh') - p['valgrind_pattern'] = stage_info.get('valgrind_pattern', - 'unit-test-*memcheck.xml') - p['testResults'] = stage_info.get('testResults', 'test_results/*.xml') + p['always_script'] = config.get('always_script', + stage_info.get('always_script', + 'ci/unit/test_post_always.sh')) + p['valgrind_pattern'] = config.get('valgrind_pattern', + stage_info.get('valgrind_pattern', + 'unit-test-*memcheck.xml')) + p['testResults'] = config.get('testResults', + stage_info.get('testResults', + 'test_results/*.xml')) p['with_valgrind'] = with_valgrind - p['NLT'] = stage_info['NLT'] runTestData = afterTest(p, runData) runTestData.each { resultKey, data -> runData[resultKey] = data } diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 72f85e1d6..d54a34834 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -17,11 +17,17 @@ * Default 'test_results/*.xml' * * config['valgrind_pattern'] Pattern for Valgrind files. - * Default: '*.memcheck.xml' + * Default: 'unit-test-*.memcheck.xml' * * config['valgrind_stash'] Name to stash Valgrind artifacts * Required if more than one stage is * creating Valgrind reports. + * + * config['NLT'] Set to true for NLT. + * + * config['FI'] Set to true for Fault Injection testing. + * FI also set NLT to true. + * */ /* groovylint-disable-next-line MethodSize */ void call(Map config = [:]) { @@ -29,9 +35,10 @@ void call(Map config = [:]) { String cbcResult = currentBuild.currentResult // Stash the Valgrind files for later analysis - String valgrind_pattern = stage_info.get('valgrind_pattern', - 'unit-test-*memcheck.xml') if (config['valgrind_stash']) { + String valgrind_pattern = config.get('valgrind_pattern', + stage_info.get('valgrind_pattern', + 'unit-test-*memcheck.xml')) try { stash name: config['valgrind_stash'], includes: valgrind_pattern } catch (hudson.AbortException e) { @@ -54,7 +61,9 @@ void call(Map config = [:]) { List artifact_list = config.get('artifacts', ['run_test.sh/*']) - String testResults = stage_info.get('testResults', 'test_results/*.xml') + String testResults = config.get('testResults', + stage_info.get('testResults', + 'test_results/*.xml')) if (testResults != 'None' ) { // groovylint-disable-next-line NoDouble double health_scale = 1.0 @@ -104,16 +113,18 @@ void call(Map config = [:]) { return } - if (stage_info['NLT']) { + Boolean fi = config.get('FI', false) + Boolean nlt = fi || config.get('NLT',stage_info.get('NLT', false)) + if (nlt) { String cb_result = currentBuild.result discoverGitReferenceBuild(referenceJob: config.get('referenceJobName', 'daos-stack/daos/master'), scm: 'daos-stack/daos', requiredResult: 'UNSTABLE') List nltTools = [issues(pattern: 'vm_test/nlt-errors.json', - name: stage_info['FI'] ? 'Fault injection' : 'NLT errors', + name: fi ? 'Fault injection' : 'NLT errors', id: sanitizedStageName() + '_VM_test')] - if (stage_info['FI']) { + if (fi) { nltTools << issues(pattern: 'nlt-client-leaks.json', name: 'Fault injection leaks', id: 'NLT_client') @@ -131,7 +142,7 @@ void call(Map config = [:]) { [threshold: 1, type: 'TOTAL_HIGH'], [threshold: 1, type: 'NEW_NORMAL', unstable: true], [threshold: 1, type: 'NEW_LOW', unstable: true]], - name: stage_info['FI']?'Fault injection':'NLT', + name: fi?'Fault injection':'NLT', tools: nltTools, scm: 'daos-stack/daos' From 6594ef781cc42f4ff22f555a5c9bdcd05aa2423e Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 15 May 2026 13:57:45 +0200 Subject: [PATCH 29/35] Pin com.diffplug.spotless plugin Signed-off-by: Tomasz Gromadzki --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 93df49941..d39e30c2e 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { id 'groovy' - id 'com.diffplug.spotless' version 'latest.release' + id 'com.diffplug.spotless' version '8.5.0' } repositories { From cc2263549aaaaec1cbf5b84d1a5daec7aa7c4c26 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 15 May 2026 14:29:32 +0200 Subject: [PATCH 30/35] Pin com.diffplug.spotless plugin 2nd Signed-off-by: Tomasz Gromadzki --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d39e30c2e..dc8e26b1f 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { id 'groovy' - id 'com.diffplug.spotless' version '8.5.0' + id 'com.diffplug.spotless' version '6.25.0' } repositories { From f5cc551c8797f6ac1cd982e20650fc36e3db66b6 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 15 May 2026 16:03:48 +0200 Subject: [PATCH 31/35] Revert "Pin com.diffplug.spotless plugin 2nd" This reverts commit cc2263549aaaaec1cbf5b84d1a5daec7aa7c4c26. Signed-off-by: Tomasz Gromadzki --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index dc8e26b1f..d39e30c2e 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { id 'groovy' - id 'com.diffplug.spotless' version '6.25.0' + id 'com.diffplug.spotless' version '8.5.0' } repositories { From b4df18e98ea39b5697fc971a4e3beaf53b26aea0 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 15 May 2026 16:04:18 +0200 Subject: [PATCH 32/35] Revert "Pin com.diffplug.spotless plugin" This reverts commit 6594ef781cc42f4ff22f555a5c9bdcd05aa2423e. Signed-off-by: Tomasz Gromadzki --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d39e30c2e..93df49941 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { id 'groovy' - id 'com.diffplug.spotless' version '8.5.0' + id 'com.diffplug.spotless' version 'latest.release' } repositories { From 5f451b4d2d47187577908888dcacdfa4d3c51ce3 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 15 May 2026 16:14:51 +0200 Subject: [PATCH 33/35] Fix: config with_valgrind override stage_info Signed-off-by: Tomasz Gromadzki --- vars/unitTestPost.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index d54a34834..0a01d3961 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -73,7 +73,7 @@ void call(Map config = [:]) { junit testResults: testResults, healthScaleFactor: health_scale } - if (stage_info['with_valgrind']) { + if (config.get('with_valgrind', stage_info.get('with_valgrind', false))) { String suite = sanitizedStageName() int vgfail = 0 String testdata From 129f2d12b5d1784703fcf88e8b8b69815ea510c4 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 18 May 2026 18:54:54 +0200 Subject: [PATCH 34/35] Fix: remove obsolete stage Doc-only: true Signed-off-by: Tomasz Gromadzki --- vars/skipStage.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/vars/skipStage.groovy b/vars/skipStage.groovy index c875a293b..381be5d0f 100644 --- a/vars/skipStage.groovy +++ b/vars/skipStage.groovy @@ -436,7 +436,6 @@ boolean call(Map config = [:]) { skip_ftest('ubuntu20', target_branch, tags) */ return true case 'Fault injection testing': - case 'NLT Fault injection testing': case 'Fault injection testing on CentOS 8': case 'Fault injection testing on EL 8': case 'Fault injection testing on EL 8.8': From f96665dbda0a43ce45729956ab284d963a969bc8 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 19 May 2026 08:48:04 +0200 Subject: [PATCH 35/35] Fix: Test results name to follow stage name Signed-off-by: Tomasz Gromadzki --- vars/unitTestPost.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/unitTestPost.groovy b/vars/unitTestPost.groovy index 0a01d3961..16d4a8a34 100755 --- a/vars/unitTestPost.groovy +++ b/vars/unitTestPost.groovy @@ -142,7 +142,7 @@ void call(Map config = [:]) { [threshold: 1, type: 'TOTAL_HIGH'], [threshold: 1, type: 'NEW_NORMAL', unstable: true], [threshold: 1, type: 'NEW_LOW', unstable: true]], - name: fi?'Fault injection':'NLT', + name: fi?'Fault injection testing':'NLT', tools: nltTools, scm: 'daos-stack/daos'