diff --git a/Jenkinsfile b/Jenkinsfile index 0a5f1cb39..cd4fee3e2 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ // That PR should be landed with out deleting the PR branch. // Then a second PR submitted to comment out the @Library line, and when it // is landed, both PR branches can be deleted. -//@Library(value='pipeline-lib@my_branch_name') _ +@Library(value='pipeline-lib@osalyk/SRE-322_new-fix') _ /* groovylint-disable-next-line CompileStatic */ job_status_internal = [:] @@ -612,7 +612,13 @@ pipeline { 'full_regression,foobar,@stages.tag@'], [tags: [[tag: 'Test-tag', value: 'datamover foobar']], tag_template: 'datamover,@stages.tag@ foobar,@stages.tag@'], - /* this one doesn't quite work due to the @commits.value@ substitution + [tags: [[tag: 'Test-tag', value: 'line1'], + [tag: 'Test-tag', value: 'line2'], + [tag: 'Test-tag', value: 'line3'], + [tag: 'Test-tag', value: 'line4'],], + tag_template: 'line1,@stages.tag@ line2,@stages.tag@ ' + + 'line3,@stages.tag@ line4,@stages.tag@'], + /* this one doesn't quite work due to the @commits.value@ substitution not accounting for the skip-list [tags: [[tag: 'Test-tag', value: 'datamover'], [tag: 'Features', value: 'foobar'], diff --git a/build.gradle b/build.gradle index 93df49941..2b267d4dd 100644 --- a/build.gradle +++ b/build.gradle @@ -17,6 +17,7 @@ dependencies { implementation localGroovy() testImplementation localGroovy() testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' + testImplementation 'org.spockframework:spock-core:2.4-groovy-4.0' testRuntimeOnly 'org.junit.platform:junit-platform-launcher:1.10.0' } diff --git a/src/test/groovy/checkTestTags.groovy b/src/test/groovy/checkTestTags.groovy new file mode 100644 index 000000000..5888a47ce --- /dev/null +++ b/src/test/groovy/checkTestTags.groovy @@ -0,0 +1,146 @@ +/* + * Copyright 2026 Hewlett Packard Enterprise Development LP + * + * SPDX-License-Identifier: BSD-2-Clause-Patent + */ + +package com.daos.pipeline + +import static org.junit.jupiter.api.Assertions.* + +import spock.lang.Specification + +class TagTemplateSpec extends Specification { + + static final String stage_name = 'Functional on Leap 15' + + private Script loadScript(String script, Map extraBinding = [:]) { + Binding binding = new Binding() + + // override bindings as required for a specific test + extraBinding.each { k, v -> + binding.setVariable(k, v) + } + + GroovyShell shell = new GroovyShell(binding) + return shell.parse(new File("vars/${script}.groovy")) + } + + private Script loadPragmasToEnv() { + Closure pragmasToMapWrap = { String commit_message -> + Script pragmasToMap = loadScript('pragmasToMap') + return pragmasToMap.call(commit_message) + } + + Map extraBinding = [ + pragmasToMap: pragmasToMapWrap, + env: [:] + ] + + return loadScript('pragmasToEnv', extraBinding) + } + + private Script loadParseStageInfo(Map extraBinding) { + extraBinding.distroVersion = { String distro -> + return 'XXA' + } + extraBinding.cachedCommitPragma = { String a, String b -> + return 'XXB' + } + extraBinding.paramsValue = { String a, String b -> + return 'XXC' + } + extraBinding.error = { String a ->} + extraBinding.getPragmaSuffix = {-> return 'XXD' } + + Closure getFunctionalStageTagsWrap = { + -> + Script getFunctionalStageTags = loadScript('getFunctionalStageTags', [ + env: extraBinding.env, + ]) + return getFunctionalStageTags.call() + } + extraBinding.getFunctionalStageTags = getFunctionalStageTagsWrap + extraBinding.startedByTimer = {-> return false } + extraBinding.branchTypeRE = { String a -> return 'XXE' } + + Closure getPragmaSuffixWrap = { + -> + Script getPragmaSuffix = loadScript('getPragmaSuffix', [ + env: extraBinding.env, + ]) + return getPragmaSuffix.call() + } + Closure envToPragmasWrap = { + -> + Script envToPragmas = loadScript('envToPragmas', [ + env: extraBinding.env, + ]) + return envToPragmas.call() + } + Closure commitPragmaWrap = {String a, String b -> + Script commitPragma = loadScript('commitPragma', [ + env: extraBinding.env, + envToPragmas: envToPragmasWrap + ]) + return commitPragma.call(a, b) + } + extraBinding.getFunctionalTags = { Map kwargs -> + Script getFunctionalTags = loadScript('getFunctionalTags', [ + getPragmaSuffix: getPragmaSuffixWrap, + getFunctionalStageTags: getFunctionalStageTagsWrap, + startedByUser: {-> return false }, + startedByUpstream: {-> return false }, + startedByTimer: {-> return false }, + commitPragma: commitPragmaWrap, + getSkippedTests: { + -> return [] + } + ]) + return getFunctionalTags.call(kwargs) + } + extraBinding.getFunctionalArgs = { LinkedHashMap a -> return [:] } + return loadScript('parseStageInfo', extraBinding) + } + + /** + * Replaces successive occurrences of the placeholder with successive tag values. + * This mirrors the behavior required by the example: each @stages.tag@ is consumed + * in order and replaced by the corresponding tags[i].value. + */ + private String expandTagTemplate(List tags) { + String cm = '''\ + Test commit\n''' + tags.each { tag -> + cm += """\ + + ${tag.tag}: ${tag.value}""" + } + Script pragmasToEnv = loadPragmasToEnv() + String tmp_pragmas = pragmasToEnv.call(cm.stripIndent()) + Map env = [ + STAGE_NAME: stage_name, + UNIT_TEST: 'true', + pragmas: tmp_pragmas, + COMMIT_MESSAGE: cm.stripIndent(), + RELEASE_BRANCH: 'master' + ] + Script parseStageInfo = loadParseStageInfo([env: env]) + Map info = parseStageInfo.call() + return info['test_tag'] + } + + def "expand tag_template by replacing @stages.tag@ with tag values in order"() { + given: "input tags and tag_template as in the example" + def tags = [ + [tag: 'Test-tag', value: 'line1'], + [tag: 'Test-tag', value: 'line2'], + [tag: 'Test-tag', value: 'line3'], + [tag: 'Test-tag', value: 'line4'], + ] + def tagTemplate = 'line1,vm line2,vm line3,vm line4,vm' + + def result = expandTagTemplate(tags) + assertEquals(tagTemplate, result) + } +} diff --git a/vars/commitPragma.groovy b/vars/commitPragma.groovy index fe9edf76d..96e4752d5 100644 --- a/vars/commitPragma.groovy +++ b/vars/commitPragma.groovy @@ -28,12 +28,35 @@ String call(String name, String def_val = null) { if (env.pragmas) { Map pragmas = envToPragmas() - if (pragmas[name.toLowerCase()]) { - return pragmas[name.toLowerCase()] - } else if (def_val) { - return def_val + String key = name.toLowerCase() + def value = pragmas[key] + + if (key == 'test-tag') { + if (value instanceof List) { + return value.join(' ') + } + if (value != null) { + return value.toString() + } + return def_val ?: '' + } + + if (value instanceof List) { + return value.join(' ') + } + if (value != null) { + return value.toString() } - return '' + return def_val ?: '' } - return commitPragmaTrusted(name, def_val) + + // fallback: trusted source + if (name.toLowerCase() == 'test-tag') { + def trusted = commitPragmaTrusted(name, def_val) + return (trusted instanceof List) ? trusted.join(' ') : (trusted ?: def_val ?: '') + } + + def trusted = commitPragmaTrusted(name, def_val) + return (trusted instanceof List) ? trusted.join(' ') : (trusted ?: def_val ?: '') } + diff --git a/vars/envToPragmas.groovy b/vars/envToPragmas.groovy index b9aade884..367d4610f 100644 --- a/vars/envToPragmas.groovy +++ b/vars/envToPragmas.groovy @@ -8,13 +8,114 @@ */ Map call() { - Map pragmas = [:] - if (env.pragmas) { - pragmas = "${env.pragmas}"[1..-2].split(', ').collectEntries { entry -> - String[] pair = entry.split('=', 2) - [(pair.first()): pair.last()] + // Work on a local copy to avoid storing non-serializable objects in env + if (!env.pragmas) { + return [:] + } + + def raw = env.pragmas + + // If someone accidentally put a Matcher into env.pragmas, convert it immediately + if (raw instanceof java.util.regex.Matcher) { + raw = raw.find() ? raw.group(0) : raw.toString() + } + + // If already a Map, normalize keys and ensure list types for test-tag + if (raw instanceof Map) { + Map m = [:] + ((Map) raw).each { k, v -> + m[k.toString().toLowerCase()] = v + } + if (m['test-tag'] != null && !(m['test-tag'] instanceof List)) { + m['test-tag'] = safeUtils.ensureList(m['test-tag']) } + return m } - return pragmas + // Convert raw to a trimmed String representation for parsing + String s = (raw instanceof String) ? raw.trim() : + (raw instanceof List) ? raw.collect { it?.toString()?.trim() }.join(',') : + raw?.toString()?.trim() + + if (!s) { + return [:] + } + + try { + // 1) Form like: [test-tag:[line1, line2, line3]] + def m1 = (s =~ /^\s* +\[([^\: +\[\] +]+)\s*:\s* +\[([^\] +]*)\] +\s*\] +$/) + + if (m1.matches()) { + def rawKey = m1[0][1] + String key = rawKey?.toString()?.trim()?.toLowerCase() ?: '' + def innerVal = m1[0][2] + String inner = innerVal?.toString()?.trim() ?: '' + List values = inner ? inner.split(/\s*,\s*/).collect { it?.toString()?.trim() } : [] + return [(key): values] + } + + // 2) Generic map-like forms: {k=v, k2=[a,b], k3:val} + String body = s + if ((body.startsWith('{') && body.endsWith('}')) || (body.startsWith('[') && body.endsWith(']'))) { + body = (body.length() > 2) ? body[1..-2].trim() : '' + } + + Map pragmasMap = [:] + def parts = [] + int depth = 0 + StringBuilder cur = new StringBuilder() + body.each { ch -> + if (ch == '[' || ch == '{') { depth++ } + if (ch == ']' || ch == '}') { depth-- } + if (ch == ',' && depth == 0) { + parts << cur.toString() + cur.setLength(0) + } else { + cur.append(ch) + } + } + if (cur.length() > 0) { parts << cur.toString() } + + parts.each { entry -> + if (!entry) return + def kv = entry.split(/[:=]/, 2) + if (kv.length == 0) return + String key = kv[0]?.toString()?.trim()?.toLowerCase() ?: '' + String rawVal = (kv.length > 1) ? kv[1] : '' + + if (rawVal instanceof List) { + pragmasMap[key] = rawVal.collect { it?.toString()?.trim() } + } else { + String rawValStr = rawVal?.toString()?.trim() ?: '' + String rv = rawValStr + if (rv.startsWith('[') && rv.endsWith(']')) { + String inner = (rv.length() > 2) ? rv[1..-2].trim() : '' + List values = inner ? inner.split(/\s*,\s*/).collect { it?.toString()?.trim() } : [] + pragmasMap[key] = values + } else { + String scalar = rv + if ((scalar.startsWith('"') && scalar.endsWith('"')) || (scalar.startsWith("'") && scalar.endsWith("'"))) { + scalar = scalar[1..-2] + } + pragmasMap[key] = scalar.trim() + } + } + } + + // Ensure test-tag is a list + if (pragmasMap['test-tag'] != null && !(pragmasMap['test-tag'] instanceof List)) { + pragmasMap['test-tag'] = safeUtils.ensureList(pragmasMap['test-tag']) + } + + return pragmasMap + } catch (Exception e) { + return [:] + } } diff --git a/vars/getFunctionalTags.groovy b/vars/getFunctionalTags.groovy index 25ec16ceb..f91b4331a 100644 --- a/vars/getFunctionalTags.groovy +++ b/vars/getFunctionalTags.groovy @@ -12,12 +12,13 @@ * default_tags launch.py tag argument to use when no parameter or commit pragma tags exist * @return String of test tags to run in the stage */ -Map call(Map kwargs = [:]) { +String call(Map kwargs = [:]) { /* groovylint-disable-next-line UnnecessaryGetter */ String pragma_suffix = kwargs.get('pragma_suffix', getPragmaSuffix()) /* groovylint-disable-next-line UnnecessaryGetter */ String stage_tags = kwargs.get('stage_tags', getFunctionalStageTags()) - String default_tags = kwargs.get('default_tags', 'pr') + /* default_tags always treated as String (commitPragma already normalizes) */ + String default_tags = kwargs.get('default_tags', 'pr')?.toString() String requested_tags = '' // Define the test tags to use in this stage @@ -30,10 +31,15 @@ Map call(Map kwargs = [:]) { requested_tags = default_tags } // Builds started from a commit should first use any commit pragma 'Test-tag*:' tags if defined - requested_tags = requested_tags ?: commitPragma('Test-tag' + pragma_suffix, commitPragma('Test-tag', '')) + if (!requested_tags) { + requested_tags = commitPragma('Test-tag' + pragma_suffix, + commitPragma('Test-tag', '')) + } - // Builds started from a commit should finally use the default tags for the stage - requested_tags = requested_tags ?: default_tags + // Fallback to default tags + if (!requested_tags) { + requested_tags = default_tags + } // Append any commit pragma 'Features:' tags if defined String features = commitPragma('Features', '') diff --git a/vars/parseStageInfo.groovy b/vars/parseStageInfo.groovy index bf7719c60..959bab120 100755 --- a/vars/parseStageInfo.groovy +++ b/vars/parseStageInfo.groovy @@ -215,20 +215,21 @@ Map call(Map config = [:]) { result['test'] = 'Functional' result['node_count'] = 9 result['always_script'] = config.get('always_script', 'ci/functional/job_cleanup.sh') + if (stage_name.contains('Hardware')) { ftest_arg_nvme = get_default_nvme() + if (stage_name.contains('Small')) { result['node_count'] = 3 } else if (stage_name.contains('Medium')) { result['node_count'] = 5 + if (stage_name.contains('Provider')) { if (stage_name.contains('Verbs')) { ftest_arg_provider = 'ofi+verbs' - } - else if (stage_name.contains('UCX')) { + } else if (stage_name.contains('UCX')) { ftest_arg_provider = 'ucx+dc_x' - } - else if (stage_name.contains('TCP')) { + } else if (stage_name.contains('TCP')) { ftest_arg_provider = 'ofi+tcp' } } @@ -236,17 +237,32 @@ Map call(Map config = [:]) { result['node_count'] = 24 } } + if (stage_name.contains('with Valgrind')) { result['with_valgrind'] = 'memcheck' config['test_tag'] = 'memcheck' } + result['pragma_suffix'] = getPragmaSuffix() // Get the ftest tags Map kwargs = [:] kwargs['pragma_suffix'] = result['pragma_suffix'] kwargs['stage_tags'] = getFunctionalStageTags() - kwargs['default_tags'] = config['test_tag'] + + def dt = config['test_tag'] + + if (!dt && config['tags'] instanceof List) { + def testTagValues = config['tags'] + .findAll { it.tag?.toLowerCase() == 'test-tag' } + .collect { it.value?.toString() } + if (testTagValues && !testTagValues.isEmpty()) { + dt = testTagValues + } + } + + kwargs['default_tags'] = (dt instanceof List) ? dt : (dt ? [dt.toString()] : []) + if (!kwargs['default_tags']) { if (startedByTimer() && env.BRANCH_NAME =~ branchTypeRE('weekly')) { kwargs['default_tags'] = 'full_regression' diff --git a/vars/pragmasToMap.groovy b/vars/pragmasToMap.groovy index edfb3ceeb..d75a9bd2f 100644 --- a/vars/pragmasToMap.groovy +++ b/vars/pragmasToMap.groovy @@ -22,7 +22,23 @@ Map call(String commit_message) { // this returns from the .each closure, not the method return } - pragmas[key.toLowerCase()] = value.trim() + String k = key.toLowerCase() + String v = value.trim() + + // Special handling: allow multiple Test-tag pragmas + if (k == 'test-tag') { + def existing = pragmas[k] + if (existing == null) { + pragmas[k] = [v] + } else if (existing instanceof List) { + pragmas[k] << v + } else { + pragmas[k] = [existing, v] + } + } else { + // default behavior for all other pragmas + pragmas[k] = v + } /* groovylint-disable-next-line CatchArrayIndexOutOfBoundsException */ } catch (ArrayIndexOutOfBoundsException ignored) { // ignore and move on to the next line diff --git a/vars/safeUtils.groovy b/vars/safeUtils.groovy new file mode 100644 index 000000000..5841deb49 --- /dev/null +++ b/vars/safeUtils.groovy @@ -0,0 +1,34 @@ +/* + * Global helper available to pipeline scripts (placed in vars so no import needed). + * Exposes ensureList and safeTrim to normalize values safely. + */ + +def call() { return this } + +/** + * Ensure the value is returned as a List. + */ +def ensureList(def v) { + if (v == null) return [] + if (v instanceof List) { + return v.collect { it == null ? '' : it.toString().trim() } + } + if (v instanceof String) { + String s = v.trim() + if (s == '') return [] + if (s.contains(',')) { + return s.split(/\s*,\s*/).collect { it.trim() } + } + return [s] + } + return [v.toString().trim()] +} + +/** + * Return a trimmed String if input is String-like, otherwise null. + */ +def safeTrim(def v) { + if (v == null) return null + if (v instanceof String) return v.trim() + return v.toString().trim() +} diff --git a/vars/selfUnitTest.groovy b/vars/selfUnitTest.groovy index 5be1eff70..b788a5eda 100644 --- a/vars/selfUnitTest.groovy +++ b/vars/selfUnitTest.groovy @@ -95,6 +95,11 @@ Signed-off-by: Brian J. Murrell ''' println(" expected_map = ${expected_map}") assert(result_map == expected_map) + println "env.pragmas = ${env.pragmas} (${env.pragmas?.getClass()})" + println "envToPragmas()['test-tag'] = ${envToPragmas()['test-tag']} (${envToPragmas()['test-tag']?.getClass()})" + println "commitPragma('Test-tag') = ${commitPragma('Test-tag')} (${commitPragma('Test-tag')?.getClass()})" + println "parseStageInfo()['test_tag'] = ${parseStageInfo()['test_tag']}" + println(' with overwrite=false') updatePragmas('Test-tag: should_not_overwrite', false) result_map = envToPragmas()