iscsi: replace host targetcli with VM-based iSCSI target

containers/runner/Dockerfile

@@ -21,7 +21,8 @@ RUN dnf -y update && \
     python3-rpm \
     squid \
     make \
-    openssh-clients && \
+    openssh-clients \
+    sshpass && \
     dnf -y clean all
 
 # FIXME(workaround): LIBGUESTFS_BACKEND=direct — remove when fixed upstream.

functions.sh

@@ -368,6 +368,194 @@ remove_iscsi_target() {
     fi
 }
 
+ISCSI_TARGET_SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -o PubkeyAuthentication=no"
+
+# Boot a VM that serves an iSCSI target over QEMU mcast socket networking.
+#
+# The target VM uses a Fedora cloud image prepared with virt-customize
+# (targetcli + sshd). The base image is cached so subsequent calls only
+# create a qcow2 overlay and SSH in to configure the target (~30s).
+#
+# Two NICs: a mcast socket NIC for iSCSI traffic (10.10.10.1) and a
+# SLIRP NIC for SSH access (hostfwd on a deterministic port).
+#
+# Args:
+#   $1 (wwn)       - iSCSI target IQN, e.g. iqn.2003-01.kickstart.test:foo
+#   $2 (initiator) - iSCSI initiator IQN (unused by target, reserved)
+#   $3 (tmpdir)    - test working directory; state files written here
+#   $4 (logfile)   - all stdout/stderr from setup is appended here
+#
+# State files written to $tmpdir:
+#   iscsi-target-domain  - libvirt domain name (for cleanup)
+#   iscsi-target-disk    - qcow2 overlay path (for cleanup)
+#   iscsi-mcast-port     - UDP port for mcast group (for additional_runner_args)
+#   iscsi-ssh-port       - TCP port for SSH hostfwd (for validate)
+#
+# Environment:
+#   KSTEST_ISCSI_CACHE        - cache dir (default: /var/tmp/kstest-iscsi-cache)
+#   KSTEST_ISCSI_TARGET_IMAGE - cloud image URL or local path
+#                               (default: latest Fedora Cloud from getfedora.org)
+create_iscsi_target_vm() {
+    local wwn=$1
+    # shellcheck disable=SC2034  # initiator reserved for future ACL use
+    local initiator=$2
+    local tmpdir=$3
+    local logfile=$4
+
+    local port_seed
+    port_seed=$(echo "${tmpdir}" | cksum | awk '{print $1}')
+    local mcast_port=$((10000 + port_seed % 20000))
+    local ssh_port=$((30000 + (port_seed + 1) % 20000))
+    local target_ip=10.10.10.1
+    local disk_img=${tmpdir}/iscsi-target.qcow2
+    local domain_name
+    domain_name="iscsi-target-$(basename "${tmpdir}")"
+
+    echo ${domain_name} > ${tmpdir}/iscsi-target-domain
+    echo ${disk_img} > ${tmpdir}/iscsi-target-disk
+    echo ${mcast_port} > ${tmpdir}/iscsi-mcast-port
+    echo ${ssh_port} > ${tmpdir}/iscsi-ssh-port
+
+    local cache_dir=${KSTEST_ISCSI_CACHE:-/var/tmp/kstest-iscsi-cache}
+    local base_img="${cache_dir}/iscsi-target-base.qcow2"
+
+    if [ ! -f "${base_img}" ]; then
+        mkdir -p "${cache_dir}"
+        (
+            flock -x 9
+            # Re-check after acquiring lock (another process may have created it)
+            if [ -f "${base_img}" ]; then
+                exit 0
+            fi
+            # Resolve the cloud image URL: use env override, or find the latest
+            # Fedora Cloud image from getfedora.org releases API
+            local image_url=${KSTEST_ISCSI_TARGET_IMAGE:-""}
+            if [ -z "${image_url}" ]; then
+                image_url=$(curl -sfL https://getfedora.org/releases.json | \
+                    python3 -c "import json,sys;print(next(r['link'] for r in json.load(sys.stdin) if 'Cloud' in r.get('subvariant','') and 'Generic' in r.get('subvariant','') and 'qcow2' in r.get('link','') and 'x86_64' in r.get('link','')))" 2>/dev/null) || true
+                if [ -z "${image_url}" ]; then
+                    echo "ERROR: Could not resolve Fedora Cloud image URL. Set KSTEST_ISCSI_TARGET_IMAGE." >&2
+                    exit 1
+                fi
+            fi
+
+            if [[ "${image_url}" == http* ]]; then
+                echo "Downloading iSCSI target base image..." >&2
+                curl -f -L --retry 3 -o "${cache_dir}/download.tmp.$$" "${image_url}" &>> ${logfile} || {
+                    echo "ERROR: Failed to download iSCSI target image from ${image_url}" >&2
+                    rm -f "${cache_dir}/download.tmp.$$"
+                    exit 1
+                }
+                mv "${cache_dir}/download.tmp.$$" "${cache_dir}/downloaded.$$.qcow2"
+            else
+                cp "${image_url}" "${cache_dir}/downloaded.$$.qcow2" || {
+                    echo "ERROR: Failed to copy iSCSI target image from ${image_url}" >&2
+                    exit 1
+                }
+            fi
+
+            echo "Preparing iSCSI target base image with virt-customize..." >&2
+            virt-customize -a "${cache_dir}/downloaded.$$.qcow2" \
+                --root-password password:testcase \
+                --install targetcli,NetworkManager \
+                --run-command 'systemctl enable sshd target' \
+                --run-command 'echo "PermitRootLogin yes" >> /etc/ssh/sshd_config.d/99-kstest.conf' \
+                --run-command 'systemctl disable cloud-init cloud-init-local cloud-config cloud-final 2>/dev/null; true' \
+                --selinux-relabel \
+                &>> ${logfile} || {
+                echo "ERROR: virt-customize failed" >&2
+                rm -f "${cache_dir}/downloaded.$$.qcow2"
+                exit 1
+            }
+
+            mv "${cache_dir}/downloaded.$$.qcow2" "${base_img}"
+            echo "Cached iSCSI target base image at ${base_img}" >&2
+        ) 9>"${cache_dir}/base.lock" || return 1
+    fi
+
+    qemu-img create -f qcow2 -b "${base_img}" -F qcow2 "${disk_img}" &>> ${logfile}
+
+    virt-install \
+        --name ${domain_name} \
+        --ram 2048 --vcpus 1 \
+        --disk path=${disk_img},bus=virtio \
+        --import --graphics none --noautoconsole \
+        --osinfo detect=on,require=off \
+        --seclabel type=none \
+        --network none \
+        --qemu-commandline="-netdev" \
+        "--qemu-commandline=socket,id=net0,mcast=230.0.0.1:${mcast_port},localaddr=127.0.0.1" \
+        --qemu-commandline="-device" \
+        "--qemu-commandline=virtio-net-pci,netdev=net0,addr=0x10" \
+        --qemu-commandline="-netdev" \
+        "--qemu-commandline=user,id=net1,hostfwd=tcp::${ssh_port}-:22" \
+        --qemu-commandline="-device" \
+        "--qemu-commandline=virtio-net-pci,netdev=net1,addr=0x11" \
+        &>> ${logfile}
+
+    local ssh_cmd="sshpass -p testcase ssh ${ISCSI_TARGET_SSH_OPTS} -p ${ssh_port} root@127.0.0.1"
+    local ssh_ok=false
+    for _retry in $(seq 1 120); do
+        if ${ssh_cmd} 'echo ready' &>/dev/null; then
+            ssh_ok=true
+            break
+        fi
+        sleep 1
+    done
+
+    if ! ${ssh_ok}; then
+        echo "ERROR: iSCSI target VM did not become reachable in 120s" >&2
+        return 1
+    fi
+
+    ${ssh_cmd} << SSHEOF &>> ${logfile}
+        set -e
+        IFACE=\$(ls /sys/bus/pci/devices/0000:00:10.0/net/ 2>/dev/null | head -1)
+        if [ -z "\$IFACE" ]; then
+            IFACE=\$(ip -o link show | grep -v lo | head -1 | awk -F': ' '{print \$2}')
+        fi
+        nmcli connection add type ethernet con-name iscsi-net ifname \$IFACE \
+            ipv4.method manual ipv4.addresses ${target_ip}/24 ipv6.method disabled
+        nmcli connection up iscsi-net
+
+        dd if=/dev/zero of=/root/disk bs=1M count=1 seek=10240
+
+        targetcli "/backstores/fileio create file_or_dev=/root/disk name=disk write_back=false"
+        targetcli "/iscsi create wwn=${wwn}"
+        targetcli "/iscsi/${wwn}/tpg1/luns create /backstores/fileio/disk"
+        targetcli "/iscsi/${wwn}/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1"
+        targetcli "/iscsi/${wwn}/tpg1/portals delete 0.0.0.0 3260" 2>/dev/null || true
+        targetcli "/iscsi/${wwn}/tpg1/portals delete ::0 3260" 2>/dev/null || true
+        targetcli "/iscsi/${wwn}/tpg1/portals create ${target_ip} 3260"
+        targetcli "/ saveconfig"
+
+        firewall-cmd --permanent --add-service=iscsi-target 2>/dev/null && \
+            firewall-cmd --reload 2>/dev/null || true
+SSHEOF
+
+    if [ $? -ne 0 ]; then
+        echo "ERROR: iSCSI target VM setup failed" >&2
+        return 1
+    fi
+
+    ${ssh_cmd} 'ss -tlnp | grep -q 3260' &>> ${logfile} || {
+        echo "ERROR: iSCSI target not listening on port 3260" >&2
+        return 1
+    }
+}
+
+remove_iscsi_target_vm() {
+    local domain_name=$1
+    local disk_img=$2
+    local logfile=$3
+
+    if [ -n "${domain_name}" ]; then
+        virsh destroy ${domain_name} &>> ${logfile} || true
+        virsh undefine ${domain_name} &>> ${logfile} || true
+    fi
+    rm -f ${disk_img}
+}
+
 apply_updates_image() {
     local image_url="${1}"
     local updates_dir="${2}"

iscsi-bind.ks.in

@@ -4,10 +4,12 @@
 iscsiname @KSTEST_ISCSINAME@
 iscsi --ipaddr @KSTEST_ISCSI_IP@ --port @KSTEST_ISCSI_PORT@ --target @KSTEST_ISCSI_TARGET@ --iface=@KSTEST_NETDEV1@
 
-bootloader --timeout=1
+# --location=none: non-iBFT iSCSI disks can't serve as boot devices
+bootloader --location=none
 zerombr
 clearpart --all
-autopart
+# --nohome: keep layout simple on the iSCSI disk
+autopart --nohome
 
 # for non-offload iSCSI /boot can be on iSCSI only when using iBFT,
 # so put it to local disk
@@ -20,7 +22,9 @@ keyboard us
 lang en
 timezone America/New_York
 rootpw qweqwe
-shutdown
+# poweroff instead of shutdown: explicit ACPI power-off avoids systemd
+# hanging on iSCSI session disconnect during the shutdown sequence
+poweroff
 
 %packages
 %end
@@ -33,7 +37,7 @@ function check_iscsi_session_nochroot() {
     local transport="$1"
     local target="$2"
 
-    iscsiadm -m session | egrep -q '^'${transport}':.*'${target}
+    iscsiadm -m session | grep -Eq '^'${transport}':.*'${target}
     if [[ $? -ne 0 ]]; then
         echo "*** Failed check: ${target} session using ${transport} exists" >> $SYSROOT/root/RESULT
     fi

iscsi-bind.sh

@@ -19,7 +19,32 @@
 
 # Ignore unused variable parsed out by tooling scripts as test tags metadata
 # shellcheck disable=SC2034
-TESTTYPE="knownfailure iscsi"
+TESTTYPE=${TESTTYPE:-"iscsi"}
 
 . ${KSTESTDIR}/iscsi.sh
 
+prepare() {
+    local ks=$1
+    local tmpdir=$2
+
+    local test_id
+    test_id=$(basename "${tmpdir}")
+    local lc_test_id
+    lc_test_id=$(echo "${test_id,,}" | tr -c 'a-z0-9\n' '-')
+    local wwn=iqn.2003-01.kickstart.test:${lc_test_id}
+    local initiator=iqn.2009-02.com.example:${lc_test_id}
+    local logfile=${tmpdir}/iscsi-target.log
+
+    create_iscsi_target_vm ${wwn} ${initiator} ${tmpdir} ${logfile} || return 1
+
+    # eth0 is the mcast NIC (PCI addr=0x10, iSCSI network)
+    sed -i \
+        -e "s#@KSTEST_ISCSI_IP@#10.10.10.1#g" \
+        -e "s#@KSTEST_ISCSI_PORT@#3260#g" \
+        -e "s#@KSTEST_ISCSI_TARGET@#${wwn}#g" \
+        -e "s#@KSTEST_ISCSINAME@#${initiator}#g" \
+        -e "s#@KSTEST_NETDEV1@#eth0#g" \
+        ${ks}
+
+    echo ${ks}
+}

iscsi-ordering.ks.in

@@ -0,0 +1,49 @@
+
+%ksappend repos/default.ks
+
+# INSTALLER-4044 reproducer: ignoredisk BEFORE iscsi
+# On unfixed builds, this fails with "Disk sda does not exist"
+# On fixed builds (deferred device resolution), this works
+ignoredisk --only-use=@KSTEST_ISCSI_DISK@
+
+iscsiname @KSTEST_ISCSINAME@
+iscsi --ipaddr @KSTEST_ISCSI_IP@ --port @KSTEST_ISCSI_PORT@ --target @KSTEST_ISCSI_TARGET@
+
+bootloader --location=none
+zerombr
+clearpart --all
+autopart --nohome
+
+keyboard us
+lang en
+timezone America/New_York
+rootpw qweqwe
+poweroff
+
+%packages
+%end
+
+%post --nochroot
+
+SYSROOT=/mnt/sysroot
+
+function check_iscsi_session_nochroot() {
+    local transport="$1"
+    local target="$2"
+
+    iscsiadm -m session | grep -Eq '^'${transport}':.*'${target}
+    if [[ $? -ne 0 ]]; then
+        echo "*** Failed check: ${target} session using ${transport} exists" >> $SYSROOT/root/RESULT
+    fi
+}
+
+check_iscsi_session_nochroot tcp @KSTEST_ISCSI_TARGET@
+
+%end
+
+%post
+# No error was written to /root/RESULT file, everything is OK
+if [[ ! -e /root/RESULT ]]; then
+   echo SUCCESS > /root/RESULT
+fi
+%end

iscsi-ordering.sh

@@ -0,0 +1,51 @@
+#
+# Copyright (C) 2025  Red Hat, Inc.
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# the GNU General Public License v.2, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY expressed or implied, including the implied warranties of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.  You should have received a copy of the
+# GNU General Public License along with this program; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
+# source code or documentation are not subject to the GNU General Public
+# License and may only be used or replicated with the express permission of
+# Red Hat, Inc.
+
+# INSTALLER-4044: ignoredisk before iscsi ordering test
+# Requires the anaconda fix that defers device resolution to process_kickstart()
+
+# Ignore unused variable parsed out by tooling scripts as test tags metadata
+# shellcheck disable=SC2034
+TESTTYPE=${TESTTYPE:-"iscsi"}
+
+. ${KSTESTDIR}/iscsi.sh
+
+prepare() {
+    local ks=$1
+    local tmpdir=$2
+
+    local test_id
+    test_id=$(basename "${tmpdir}")
+    local lc_test_id
+    lc_test_id=$(echo "${test_id,,}" | tr -c 'a-z0-9\n' '-')
+    local wwn=iqn.2003-01.kickstart.test:${lc_test_id}
+    local initiator=iqn.2009-02.com.example:${lc_test_id}
+    local logfile=${tmpdir}/iscsi-target.log
+
+    create_iscsi_target_vm ${wwn} ${initiator} ${tmpdir} ${logfile} || return 1
+
+    # sda is the expected device name for the first SCSI disk from iSCSI on x86_64
+    sed -i \
+        -e "s#@KSTEST_ISCSI_IP@#10.10.10.1#g" \
+        -e "s#@KSTEST_ISCSI_PORT@#3260#g" \
+        -e "s#@KSTEST_ISCSI_TARGET@#${wwn}#g" \
+        -e "s#@KSTEST_ISCSINAME@#${initiator}#g" \
+        -e "s#@KSTEST_ISCSI_DISK@#sda#g" \
+        ${ks}
+
+    echo ${ks}
+}