domainmgr, zedmanager, zedagent: SMT/NUMA-aware CPU placement for pinned applications

evetest/broker/broker.go

@@ -1120,6 +1120,7 @@ func (b *broker) SetupDevices(
 
 		// Build EVE device specification for the provider.
 		dev.CPUs = uint(eveDevice.GetCpus())
+		dev.ThreadsPerCore = uint(eveDevice.GetThreadsPerCore())
 		dev.MemoryBytes = eveDevice.GetMemoryBytes()
 		dev.WithTPM = eveDevice.GetWithTpm()
 		dev.SerialNumber = eveDevice.GetSerialNumber()

evetest/broker/provider/common.go

@@ -18,6 +18,35 @@ import (
 	"github.com/vishvananda/netlink"
 )
 
+// CPUTopology converts a device spec's CPU count and requested threads-per-core
+// into a concrete sockets/cores/threads layout for the hypervisor.
+//
+// A guest can only reason about SMT if it is told which logical CPUs share a
+// physical core, and neither QEMU nor libvirt infers that: left alone both
+// present every CPU as its own single-thread core. Keeping the arithmetic here
+// means the qemu and libvirt providers cannot disagree about it.
+//
+// The default (threads-per-core of zero or one) reproduces exactly what the
+// providers did before this existed: one socket, one core per CPU, one thread
+// per core. A requested value that does not divide the CPU count is ignored
+// rather than silently rounded, since a partial core is not a thing a guest can
+// be shown.
+func CPUTopology(cpus, threadsPerCore uint) (sockets, cores, threads uint) {
+	if threadsPerCore < 2 || cpus%threadsPerCore != 0 {
+		return 1, cpus, 1
+	}
+	return 1, cpus / threadsPerCore, threadsPerCore
+}
+
+// smpArg renders the QEMU -smp value for a device's CPU layout. QEMU expands a
+// bare "-smp N" with threads=1, so the topology has to be stated explicitly for
+// the guest to see sibling threads at all.
+func smpArg(cpus, threadsPerCore uint) string {
+	sockets, cores, threads := CPUTopology(cpus, threadsPerCore)
+	return fmt.Sprintf("%d,sockets=%d,cores=%d,threads=%d",
+		cpus, sockets, cores, threads)
+}
+
 // namePrefix is prepended to all domain (VM) names created by a device
 // provider. This ensures that test/dev resources are clearly separated from
 // user-managed objects. When listing devices, the prefix is stripped so that

evetest/broker/provider/cputopology_test.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2026 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package provider
+
+import "testing"
+
+func TestCPUTopology(t *testing.T) {
+	tests := []struct {
+		name                            string
+		cpus, threadsPerCore            uint
+		wantSockets, wantCores, wantThr uint
+	}{
+		// The default must reproduce exactly what the providers did before
+		// threads-per-core existed: every CPU its own single-thread core.
+		{"unset is one thread per core", 8, 0, 1, 8, 1},
+		{"explicit 1 is one thread per core", 8, 1, 1, 8, 1},
+
+		{"SMT halves the core count", 8, 2, 1, 4, 2},
+		{"SMT with the minimum core count", 2, 2, 1, 1, 2},
+
+		// A partial core cannot be presented to a guest, so an indivisible
+		// request degrades to the default rather than being rounded.
+		{"odd CPU count cannot be SMT", 7, 2, 1, 7, 1},
+		{"four threads per core", 8, 4, 1, 2, 4},
+		{"threads exceeding CPUs degrade", 2, 4, 1, 2, 1},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sockets, cores, threads := CPUTopology(tt.cpus, tt.threadsPerCore)
+			if sockets != tt.wantSockets || cores != tt.wantCores || threads != tt.wantThr {
+				t.Errorf("CPUTopology(%d, %d) = %d/%d/%d, want %d/%d/%d",
+					tt.cpus, tt.threadsPerCore, sockets, cores, threads,
+					tt.wantSockets, tt.wantCores, tt.wantThr)
+			}
+			// However the CPUs are arranged, the guest must still end up with
+			// the number of logical CPUs it asked for.
+			if got := sockets * cores * threads; got != tt.cpus {
+				t.Errorf("topology describes %d logical CPUs, want %d", got, tt.cpus)
+			}
+		})
+	}
+}
+
+func TestSMPArg(t *testing.T) {
+	if got, want := smpArg(8, 0), "8,sockets=1,cores=8,threads=1"; got != want {
+		t.Errorf("smpArg(8, 0) = %q, want %q", got, want)
+	}
+	if got, want := smpArg(8, 2), "8,sockets=1,cores=4,threads=2"; got != want {
+		t.Errorf("smpArg(8, 2) = %q, want %q", got, want)
+	}
+}

evetest/broker/provider/libvirt.go

@@ -410,11 +410,14 @@ func (p *LibvirtProvider) SetupDevice(
 		},
 		CPU: &libvirtxml.DomainCPU{
 			Mode: "host-passthrough", // expose host CPU features
-			Topology: &libvirtxml.DomainCPUTopology{
-				Sockets: 1,
-				Cores:   int(spec.CPUs),
-				Threads: 1,
-			},
+			Topology: func() *libvirtxml.DomainCPUTopology {
+				sockets, cores, threads := CPUTopology(spec.CPUs, spec.ThreadsPerCore)
+				return &libvirtxml.DomainCPUTopology{
+					Sockets: int(sockets),
+					Cores:   int(cores),
+					Threads: int(threads),
+				}
+			}(),
 		},
 		Features:        features,
 		Devices:         devices,

evetest/broker/provider/provider.go

@@ -173,6 +173,13 @@ type DeviceSpec struct {
 	// CPUs is the number of CPU cores to allocate.
 	CPUs uint
 
+	// ThreadsPerCore is how many SMT hardware threads each physical core
+	// exposes to the guest. Zero or one -- the default -- gives every CPU its
+	// own single-thread core. Higher values arrange the same number of CPUs as
+	// CPUs/ThreadsPerCore cores with sibling threads, which a guest needs
+	// before it can make SMT-aware placement decisions.
+	ThreadsPerCore uint
+
 	// MemoryBytes is the amount of RAM in bytes.
 	MemoryBytes uint64
 

evetest/broker/provider/qemu.go

@@ -1490,7 +1490,7 @@ func (dev *qemuDevice) buildArgs() []string {
 		"-enable-kvm",
 		"-machine", "q35",
 		"-cpu", "host",
-		"-smp", fmt.Sprintf("%d", dev.spec.CPUs),
+		"-smp", smpArg(dev.spec.CPUs, dev.spec.ThreadsPerCore),
 		"-m", fmt.Sprintf("%d", dev.spec.MemoryBytes>>20),
 		"-nographic",
 	}

evetest/devconfig.go

@@ -903,9 +903,46 @@ type ApplicationInstanceConfig struct {
 	// rewire) the app's VolumeRefList to point at it; they never create or
 	// remove the volume itself.
 	Mounts []MountConfig
+	// CPUPlacement is the application's CPU placement intent. Its zero value
+	// sends no policy at all, so an application that does not set it keeps
+	// today's best-effort placement.
+	CPUPlacement CPUPlacementConfig
+	// StartDelayInSeconds holds this application back for that long while the
+	// others start normally.
+	//
+	// The delay is counted from the moment EVE first obtains configuration
+	// after booting, not from when this field is set, so setting it on an
+	// already-running application changes nothing until the next boot. That is
+	// what makes it usable as a test instrument: it is the only way to give a
+	// device a deterministic, staggered application start order without
+	// changing which applications are configured.
+	StartDelayInSeconds uint32
 	// Many more parameters can be configured; they will be added later as needed.
 }
 
+// CPUPlacementConfig expresses what an application needs from CPU placement:
+// whether it gets host CPUs of its own, whether those must be whole physical
+// cores, how its vCPUs map onto SMT siblings, and where the hypervisor's
+// emulator/IO threads run.
+//
+// The device picks the concrete host CPUs -- the controller only states intent
+// -- so there is deliberately no way here to name specific CPU ids. Assertions
+// about which CPUs a workload actually got are made against what the device
+// reports back, or read off the device directly.
+type CPUPlacementConfig struct {
+	// Policy is the on-switch: CPU_POLICY_DEDICATED activates the rest.
+	Policy        eveconfig.CpuPolicy
+	FullPCPUsOnly bool
+	// ThreadsPerCore selects how many SMT siblings of each dedicated core
+	// become vCPUs: 2 (whole-core-SMT, the default when unset) or 1
+	// (one-per-core, sibling parked idle).
+	ThreadsPerCore   uint32
+	NUMAPolicy       eveconfig.NumaPolicy
+	IOPlacement      eveconfig.IoPlacement
+	IsolationTier    eveconfig.IsolationTier
+	DisruptionPolicy eveconfig.DisruptionPolicy
+}
+
 // MountConfig attaches an existing volume to an application, in addition to
 // its root disk. The volume must already exist in the device configuration
 // (created via EdgeDeviceConfig.AddVolume or AddBlankVolume) --
@@ -942,6 +979,13 @@ func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string,
 		VncPasswd:                    config.VNCPassword,
 		DisableLogs:                  config.DisableLogs,
 		EnforceNetworkInterfaceOrder: config.EnforceNetIntfOrder,
+		CpuPolicy:                    config.CPUPlacement.Policy,
+		FullPcpusOnly:                config.CPUPlacement.FullPCPUsOnly,
+		ThreadsPerCore:               config.CPUPlacement.ThreadsPerCore,
+		NumaPolicy:                   config.CPUPlacement.NUMAPolicy,
+		IoPlacement:                  config.CPUPlacement.IOPlacement,
+		IsolationTier:                config.CPUPlacement.IsolationTier,
+		DisruptionPolicy:             config.CPUPlacement.DisruptionPolicy,
 	}
 	if config.MemoryBytes != 0 {
 		vmConfig.Memory = uint32(config.MemoryBytes / KiB)
@@ -951,11 +995,12 @@ func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string,
 			Uuid:    appUUID.String(),
 			Version: "1",
 		},
-		Displayname:    config.DisplayName,
-		Fixedresources: vmConfig,
-		Activate:       config.Activate,
-		ProfileList:    config.ProfileList,
-		RemoteConsole:  config.RemoteConsole,
+		Displayname:         config.DisplayName,
+		Fixedresources:      vmConfig,
+		Activate:            config.Activate,
+		ProfileList:         config.ProfileList,
+		RemoteConsole:       config.RemoteConsole,
+		StartDelayInSeconds: config.StartDelayInSeconds,
 	}
 	if volumeUUID != NilUUID {
 		appInstConfig.VolumeRefList = append(appInstConfig.VolumeRefList,
@@ -2391,7 +2436,7 @@ func (dc *EdgeDeviceConfig) buildMountRefs(
 func (dc *EdgeDeviceConfig) UpdateApplication(
 	appUUID uuid.UUID, newConfig ApplicationInstanceConfig) {
 	// For now, we will only allow to change Activation flag, profile list,
-	// adapters, and mounts.
+	// adapters, mounts, and the start delay.
 	for i, app := range dc.Apps {
 		if app.Uuidandversion.Uuid == appUUID.String() {
 			newProtoConfig := newConfig.toProto(dc.th, dc.DeviceName, appUUID, NilUUID)
@@ -2442,6 +2487,10 @@ func (dc *EdgeDeviceConfig) UpdateApplication(
 				app.Restart.Counter++
 			}
 			dc.Apps[i].Activate = newProtoConfig.Activate
+			// The start delay is not a fixed resource: it only decides when the
+			// application is released on the next boot, so it can be changed on
+			// a deployed application without a purge or a restart.
+			dc.Apps[i].StartDelayInSeconds = newProtoConfig.StartDelayInSeconds
 			dc.Apps[i].ProfileList = newProtoConfig.ProfileList
 			dc.Apps[i].Adapters = newProtoConfig.Adapters
 			dc.Apps[i].Interfaces = newProtoConfig.Interfaces

evetest/go.mod

@@ -100,3 +100,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/lf-edge/eve-api/go => github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc

evetest/go.sum

@@ -107,8 +107,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
-github.com/lf-edge/eve-api/go v0.0.0-20260812180240-99d02ddcfcb0 h1:vFrZPWDaFupy4K1fYIRDJOnW8aaGNMZ1/1DkNRXrZe0=
-github.com/lf-edge/eve-api/go v0.0.0-20260812180240-99d02ddcfcb0/go.mod h1:6HxNA/qKJVEqwpuOFkcQ0h3QyotvAs/cjHLC961FPOY=
 github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d h1:tUBb9M6u42LXwHAYHyh22wJeUUQlTpDkXwRXalpRqbo=
 github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d/go.mod h1:Nn3juMJJ1G8dyHOebdZyS4jOB/fuxAd5fIajBaWjHr8=
 github.com/lf-edge/eve/pkg/pillar v0.0.0-20260421125048-8d3825045e4e h1:MGhvM4TrXHYyssTTcEkpc79zatoZ1oJRrrbjTU++ps0=
@@ -149,6 +147,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc h1:8NwHnhH0EmLCnnicKa00cHP0maYbryuy4Up11il0I9A=
+github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc/go.mod h1:6HxNA/qKJVEqwpuOFkcQ0h3QyotvAs/cjHLC961FPOY=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
 github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=

evetest/grpcapi/proto/common.proto

@@ -93,6 +93,13 @@ message EVEDevice {
   string serial_number = 5;
   repeated EVEInterface interfaces = 6;
   ImageRef image = 7;
+  // How many SMT threads each physical core of the device exposes.
+  // Zero or one means no SMT: the device sees `cpus` single-thread cores,
+  // which is the default. Two makes the device see cpus/2 physical cores
+  // with two hardware threads each, which is what a test exercising
+  // SMT-aware CPU placement needs -- without it the guest has no sibling
+  // thread to place a second vCPU on.
+  uint32 threads_per_core = 8;
 }
 
 // Represents the current lifecycle state of an EVE device.