diff --git a/evetest/testapps/logger-ctr/Dockerfile b/evetest/testapps/logger-ctr/Dockerfile new file mode 100644 index 00000000000..a46224eaed0 --- /dev/null +++ b/evetest/testapps/logger-ctr/Dockerfile @@ -0,0 +1,7 @@ +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +FROM alpine:3.21 + +COPY init.sh / +CMD ["/bin/sh", "/init.sh"] diff --git a/evetest/testapps/logger-ctr/Makefile b/evetest/testapps/logger-ctr/Makefile new file mode 100644 index 00000000000..2718ad2eb37 --- /dev/null +++ b/evetest/testapps/logger-ctr/Makefile @@ -0,0 +1,16 @@ +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +EVETEST_ORG ?= lfedge +IMAGE = $(EVETEST_ORG)/evetest-logger-ctr +# Update VERSION whenever there is a change made to this app. +VERSION ?= 1.0 + +DOCKER_TARGET ?= load +DOCKER_PLATFORM ?= $(shell uname -s | tr '[A-Z]' '[a-z]')/$(subst aarch64,arm64,$(subst x86_64,amd64,$(shell uname -m))) + +build: + docker buildx build \ + --$(DOCKER_TARGET) \ + --platform $(DOCKER_PLATFORM) \ + -t $(IMAGE):$(VERSION) . diff --git a/evetest/testapps/logger-ctr/init.sh b/evetest/testapps/logger-ctr/init.sh new file mode 100644 index 00000000000..f70757dd003 --- /dev/null +++ b/evetest/testapps/logger-ctr/init.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# Copyright (c) 2026 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# Emits a startup banner followed by a numbered heartbeat every few seconds. +# Tests match on these two message kinds to verify that EVE collects +# application stdout from container creation onwards and keeps streaming it +# to the controller. The heartbeat counter restarts from 1 on every container +# (re)creation, so counting banner occurrences tells restarts apart. + +echo "evetest-logger-ctr: started" + +i=0 +while true; do + i=$((i + 1)) + echo "evetest-logger-ctr: heartbeat $i" + sleep 5 +done diff --git a/evetest/tests/apps/applogs_test.go b/evetest/tests/apps/applogs_test.go new file mode 100644 index 00000000000..f2a5ce7e304 --- /dev/null +++ b/evetest/tests/apps/applogs_test.go @@ -0,0 +1,206 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test collection and delivery of application logs to the controller. + +package apps_test + +import ( + "regexp" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" +) + +const ( + // Banner printed once by lfedge/evetest-logger-ctr on container start. + loggerAppStartupMsg = "evetest-logger-ctr: started" + // Heartbeat printed by the same app every few seconds, with a counter + // that restarts from 1 on every (re)creation of the container. + loggerAppHeartbeatRE = `evetest-logger-ctr: heartbeat [0-9]+` +) + +// TestAppLogs verifies that EVE captures the standard output of a container +// application and delivers it to the controller as application log messages, +// both for the very first output produced at container creation and for +// output produced continuously afterwards, and that log collection resumes +// after the application is stopped and started again. +// +// The application is lfedge/evetest-logger-ctr, a minimal container that +// prints a one-off startup banner and then a numbered heartbeat every few +// seconds. The banner is what makes the restart phase verifiable without +// relying on timestamps: the count of banner occurrences must grow from one +// to two once the app has been recreated. Device and application clocks can +// differ, so counting is preferred over filtering log entries by time. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- logs travel over the device's own +// management connectivity, so a single mgmt+apps port with DHCP suffices. +// The app needs no connectivity of its own. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps). Log delivery needs no tuning: +// the framework already defaults newlog.allow.fastupload to true. +// - Local NI "local-ni" (10.11.12.0/24, gateway .1) on ethernet0. +// - Container app "logger-app" (lfedge/evetest-logger-ctr) with a single +// VIF on the NI and an allow-all ACL. No port forwarding: the test never +// enters the app, it only reads what the app printed. +// +// Phases / assertions +// ------------------- +// 1. setup-done -> config-applied -> app-is-running: the container is up. +// 2. startup-log-received: the startup banner appears in the application +// logs. This covers output produced before anything could be injected +// into the app from the outside, i.e. the container-creation log path. +// 3. heartbeat-logs-received: at least three heartbeat lines arrive, +// proving logs keep streaming rather than being captured only once. +// 4. app-stopped -> app-started: deactivate the app and wait for HALTED, +// then activate it again and wait for RUNNING. +// 5. startup-log-received-after-restart: the startup banner is now present +// twice, so the recreated container's output is being collected again. +// 6. Delete the app and wait until the device reports it gone. +// +// Log assertions poll GetAppLogs via Eventually; evetest has no channel-based +// watch for application logs yet (only WatchLogs for device logs), so a +// deliberately coarse polling interval is used. +// +// Test params +// ----------- +// - HYPERVISOR. Under Kubevirt the test waits for the cluster node to +// become ready before deploying the app. +// +// Suite placement +// --------------- +// - TestApplicationSuite (deploys an app, hence hypervisor-parameterized). +func TestAppLogs(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + // Define configurable parameters available for the test. + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() + + // Set up the test harness and specify the test prerequisites. + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + // Build and apply the device configuration. + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := addLocalNI(devConfig) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "logger-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: loggerCtrImage, + Tag: loggerCtrTag, + }, + VirtualizationMode: eveconfig.VmMode_HVM, // PV does not work in xen + CPUs: 1, + MemoryBytes: 256 * evetest.MiB, + NetworkAdapters: []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + }, + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("config-applied") + + device.WaitUntilAppIsRunning(appUUID, 10*time.Minute) + evetest.Checkpoint("app-is-running") + + const ( + logTimeout = 10 * time.Minute + logPolling = 10 * time.Second + appRestartTimeout = 5 * time.Minute + ) + + // Phase 2: output produced at container creation reaches the controller. + t.Eventually(func() int { + return countAppLogs(device, appUUID, evetest.LogMsgMatch{ + MsgHasSubstring: loggerAppStartupMsg, + }) + }, logTimeout, logPolling).Should(BeNumerically(">=", 1), + "Application startup message was not reported to the controller") + evetest.Checkpoint("startup-log-received") + + // Phase 3: logs keep streaming while the app runs. + t.Eventually(func() int { + return countAppLogs(device, appUUID, evetest.LogMsgMatch{ + MsgMatchesRegexp: *regexp.MustCompile(loggerAppHeartbeatRE), + }) + }, logTimeout, logPolling).Should(BeNumerically(">=", 3), + "Application heartbeat messages were not reported to the controller") + evetest.Checkpoint("heartbeat-logs-received") + + // Phase 4: stop and start the application. + device.DeactivateApplication(appUUID, true, appRestartTimeout) + evetest.Checkpoint("app-stopped") + device.ActivateApplication(appUUID, true, appRestartTimeout) + evetest.Checkpoint("app-started") + + // Phase 5: the recreated container prints the banner again. + t.Eventually(func() int { + return countAppLogs(device, appUUID, evetest.LogMsgMatch{ + MsgHasSubstring: loggerAppStartupMsg, + }) + }, logTimeout, logPolling).Should(BeNumerically(">=", 2), + "Application startup message was not reported again after app restart") + evetest.Checkpoint("startup-log-received-after-restart") + + // Phase 6: clean up. + deleteAppAndWait(t, device, devConfig, appUUID) +} + +// countAppLogs returns how many application log messages published so far +// match the given criteria. +func countAppLogs(device *evetest.EdgeDevice, appUUID uuid.UUID, + match evetest.LogMsgMatch) int { + return len(device.GetAppLogs(appUUID, match)) +} diff --git a/evetest/tests/apps/helpers_test.go b/evetest/tests/apps/helpers_test.go new file mode 100644 index 00000000000..9dcb8bbd87d --- /dev/null +++ b/evetest/tests/apps/helpers_test.go @@ -0,0 +1,126 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package apps_test + +import ( + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +const ( + // Logical name of the (single) edge device used by every test in this package. + devName = "edge-dev" + + // Image of the general-purpose test container (ships sshd, curl, iproute2, ...). + ubuntuCtrImage = "lfedge/evetest-ubuntu-ctr" + ubuntuCtrTag = "1.0" + + // Image of the container that prints a startup banner and periodic + // heartbeats, used by the application log test. + loggerCtrImage = "lfedge/evetest-logger-ctr" + loggerCtrTag = "1.0" + + // Local network instance shared by the tests in this package. + niDisplayName = "local-ni" + niSubnet = "10.11.12.0/24" + niGateway = "10.11.12.1" + + // Port on the edge node forwarded to the app's sshd. + appSSHFwdPort = 2222 +) + +// Credentials baked into the evetest-ubuntu-ctr image. +var appAuth = evetest.UsernamePasswordAuth{ + Username: "root", + Password: "testpassword", +} + +// addLocalNI adds the local network instance shared by the tests in this +// package: 10.11.12.0/24 on ethernet0, which is also where the metadata +// server (169.254.169.254) is reachable from connected apps. +func addLocalNI(devConfig *evetest.EdgeDeviceConfig) uuid.UUID { + return devConfig.AddNetworkInstance(evetest.LocalNetworkInstanceConfig{ + DisplayName: niDisplayName, + Port: "ethernet0", + Subnet: evetest.IPSubnet(niSubnet), + DHCPRange: types.IPRange{ + Start: evetest.IPAddress("10.11.12.2"), + End: evetest.IPAddress("10.11.12.254"), + }, + Gateway: evetest.IPAddress(niGateway), + MTU: 1500, + }) +} + +// singleVIFWithSSH describes the app network adapter used by tests that need +// to run commands inside the application: a VIF on the local NI, the sshd +// port forwarded from the edge node, and an allow-all ACL (needed among other +// things to reach the metadata server). +func singleVIFWithSSH(niUUID uuid.UUID) []evetest.AppNetworkAdapter { + return []evetest.AppNetworkAdapter{ + evetest.VirtualNetworkAdapter{ + LogicalLabel: "vif0", + NetworkInstanceUUID: niUUID, + PortFwdRules: []evetest.PortFwdRule{ + { + Protocol: evetest.NetworkProtocolTCP, + EdgeNodePort: appSSHFwdPort, + AppPort: 22, + }, + }, + ACLAllowRules: []evetest.ACLAllowRule{ + { + Protocol: evetest.NetworkProtocolAny, + RemoteSubnet: evetest.IPSubnet("0.0.0.0/0"), + }, + }, + }, + } +} + +// deleteAppAndWait removes the application from the device configuration, +// applies it, and blocks until the device reports the instance as gone. Tests +// wait for this rather than just applying the deletion, so that teardown +// cannot race the device-config reset performed before the next test in the +// suite. +func deleteAppAndWait(t *WithT, device *evetest.EdgeDevice, + devConfig *evetest.EdgeDeviceConfig, appUUID uuid.UUID) { + const timeout = 5 * time.Minute + appUpdates, stopAppWatch := device.WatchAppInfo(appUUID) + defer stopAppWatch() + devConfig.DeleteApplication(appUUID) + device.ApplyConfig(devConfig, false, false) + t.Eventually(appUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "Application instance is deleted", + func(info *eveinfo.ZInfoApp) bool { + return info.State == eveinfo.ZSwState_INVALID + }))) +} + +// waitForAppSSH blocks until commands can be executed inside the application +// over the port-forwarded sshd. Reaching RUNNING only means the domain was +// created; sshd inside the container needs more time to accept connections. +func waitForAppSSH(t *WithT, device *evetest.EdgeDevice, appUUID uuid.UUID) { + const ( + sshTimeout = 20 * time.Second + timeout = 3 * time.Minute + polling = 5 * time.Second + ) + evetest.Logger().Infof("Waiting for app %q SSH to become reachable...", appUUID) + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "echo hello", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(output).To(ContainSubstring("hello")) + }, timeout, polling).Should(Succeed()) +} diff --git a/evetest/tests/apps/metadata_test.go b/evetest/tests/apps/metadata_test.go new file mode 100644 index 00000000000..41a01f4d4b8 --- /dev/null +++ b/evetest/tests/apps/metadata_test.go @@ -0,0 +1,207 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test the application metadata channel served by EVE at 169.254.169.254. + +package apps_test + +import ( + "fmt" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// Base URL of EVE's metadata server as seen from inside an application. +const metadataServerURL = "http://169.254.169.254/eve/v1" + +// TestAppInstanceMetadata verifies EVE's application metadata channel: an +// application POSTs a payload to the link-local metadata server +// (169.254.169.254) and EVE must forward it to the controller as a +// ZInfoAppInstMetaData info message tagged with the application UUID and with +// the metadata type matching the endpoint that was used. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- the metadata server is served on the +// local network instance bridge, so a single mgmt+apps port with DHCP is +// all that is required. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps). +// - Local NI "local-ni" (10.11.12.0/24, gateway .1) on ethernet0. +// - Container app "metadata-app" (lfedge/evetest-ubuntu-ctr) with a single +// VIF on the NI, port-fwd 2222->22 so the test can drive curl from inside +// the app, and an allow-all ACL -- the metadata server is subject to ACL +// filtering like any other destination. +// +// Phases / assertions +// ------------------- +// 1. setup-done -> config-applied -> app-is-running -> app-ssh-reachable: +// the container is up and reachable over the port-forwarded sshd. +// 2. kubeconfig-metadata-published: with the WatchAppMetadata subscription +// opened first (EVE publishes the info message as soon as it receives the +// POST), POST {"hello":"world"} to /eve/v1/kubeconfig. Assert that a +// ZInfoAppInstMetaData arrives for this app with +// Type=APP_INST_META_DATA_TYPE_KUBE_CONFIG and Data byte-for-byte equal +// to the posted body. +// 3. custom-status-metadata-published: POST a distinct payload to +// /eve/v1/app/appCustomStatus and assert the same, this time with +// Type=APP_INST_META_DATA_TYPE_CUSTOM_STATUS. This proves the reported +// metadata type follows the endpoint rather than being hardcoded. +// 4. GET /eve/v1/hostname returns the application instance UUID -- a +// read-only sanity check that the same server also serves the app its own +// identity. +// 5. Delete the app and wait until the device reports it gone. +// +// The POSTs are wrapped in Eventually: reaching RUNNING (and even being +// SSH-reachable) does not guarantee that zedrouter has already attached the +// metadata HTTP handler to this app's VIF. +// +// Test params +// ----------- +// - HYPERVISOR. Under Kubevirt the test waits for the cluster node to +// become ready before deploying the app. +// +// Suite placement +// --------------- +// - TestApplicationSuite (deploys an app, hence hypervisor-parameterized). +func TestAppInstanceMetadata(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + // Define configurable parameters available for the test. + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() + + // Set up the test harness and specify the test prerequisites. + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + // Build and apply the device configuration. + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := addLocalNI(devConfig) + appUUID := devConfig.AddApplication(evetest.ApplicationInstanceConfig{ + DisplayName: "metadata-app", + Activate: true, + Image: evetest.DockerContainer{ + ImageName: ubuntuCtrImage, + Tag: ubuntuCtrTag, + }, + VirtualizationMode: eveconfig.VmMode_HVM, // PV does not work in xen + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: singleVIFWithSSH(niUUID), + }) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("config-applied") + + device.WaitUntilAppIsRunning(appUUID, 10*time.Minute) + evetest.Checkpoint("app-is-running") + + waitForAppSSH(t, device, appUUID) + evetest.Checkpoint("app-ssh-reachable") + + const ( + sshTimeout = 20 * time.Second + metadataTimeout = 5 * time.Minute + ) + + // Subscribe before publishing anything - EVE reports the metadata to the + // controller immediately after the POST is accepted. + metaUpdates, stopMetaWatch := device.WatchAppMetadata(appUUID) + + // Phase 2: kubeconfig endpoint. + const kubeconfigPayload = `{"hello":"world"}` + postAppMetadata(t, device, appUUID, "/kubeconfig", kubeconfigPayload, + metadataTimeout, sshTimeout) + t.Eventually(metaUpdates, metadataTimeout).Should(Receive(matchers.SatisfyPredicate( + "Kubeconfig metadata posted by the app is reported to the controller", + func(meta *eveinfo.ZInfoAppInstMetaData) bool { + return meta.GetType() == + eveinfo.AppInstMetaDataType_APP_INST_META_DATA_TYPE_KUBE_CONFIG && + string(meta.GetData()) == kubeconfigPayload + }))) + evetest.Checkpoint("kubeconfig-metadata-published") + + // Phase 3: appCustomStatus endpoint (different metadata type). + const customStatusPayload = `{"status":"evetest-custom-status"}` + postAppMetadata(t, device, appUUID, "/app/appCustomStatus", customStatusPayload, + metadataTimeout, sshTimeout) + t.Eventually(metaUpdates, metadataTimeout).Should(Receive(matchers.SatisfyPredicate( + "Custom status posted by the app is reported to the controller", + func(meta *eveinfo.ZInfoAppInstMetaData) bool { + return meta.GetType() == + eveinfo.AppInstMetaDataType_APP_INST_META_DATA_TYPE_CUSTOM_STATUS && + string(meta.GetData()) == customStatusPayload + }))) + evetest.Checkpoint("custom-status-metadata-published") + + // Phase 4: the metadata server tells the app its own identity. + hostname, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "curl -sS "+metadataServerURL+"/hostname", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(strings.TrimSpace(hostname)).To(Equal(appUUID.String())) + + // Phase 5: clean up. + stopMetaWatch() + deleteAppAndWait(t, device, devConfig, appUUID) +} + +// postAppMetadata POSTs payload to the given metadata server endpoint from +// inside the application, retrying until the server answers with 200. +func postAppMetadata(t *WithT, device *evetest.EdgeDevice, appUUID uuid.UUID, + endpoint, payload string, timeout, sshTimeout time.Duration) { + curl := fmt.Sprintf( + "curl -sS -o /dev/null -w '%%{http_code}' "+ + "-H 'Content-Type: application/json' --request POST -d '%s' %s%s", + payload, metadataServerURL, endpoint) + evetest.Logger().Infof("Publishing app metadata via %s", endpoint) + t.Eventually(func(t Gomega) { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + curl, sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(strings.TrimSpace(output)).To(Equal("200")) + }, timeout, 5*time.Second).Should(Succeed()) +} diff --git a/evetest/tests/apps/testsuite_test.go b/evetest/tests/apps/testsuite_test.go index a589b996af3..824822be9f7 100644 --- a/evetest/tests/apps/testsuite_test.go +++ b/evetest/tests/apps/testsuite_test.go @@ -11,10 +11,15 @@ import ( // TestAppsSuite drives application-lifecycle scenarios that are not // specifically about networking (regression tests for zedmanager/volumemgr -// bugs, VNC console access, etc.) -- kept separate from +// bugs, VNC console access, how configuration and data flow between the +// controller, EVE and a running application) -- kept separate from // evetest/tests/networking's TestApplicationConnectivitySuite, which is // already large and focused on network connectivity. // +// Every subtest deploys at least one application and therefore shares the +// HYPERVISOR parameter -- the suite declares evetest.HypervisorParameter() +// once and every subtest reads it via evetest.GetHypervisorParameterValue(). +// // Subtests // -------- // - TestPurgeNeverActivatedApp -- regression test for a zedmanager bug @@ -22,6 +27,13 @@ import ( // would leave it stuck instead of recovering. // - TestVNC -- VNC access to a VM app, a container app, and the container // app's shim VM console. +// - TestAppInstanceMetadata -- app posts metadata to the link-local +// metadata server; EVE reports it to the controller. +// - TestAppUserData -- plain key=value user-data becomes container +// environment; cloud-config write_files is applied once per user-data +// version and survives an app restart. +// - TestAppLogs -- application stdout is collected and delivered to the +// controller, including after the app is stopped and started again. func TestAppsSuite(test *testing.T) { evetest.Init(test) defer evetest.Close() @@ -37,5 +49,14 @@ func TestAppsSuite(test *testing.T) { evetest.TestCase{ Test: TestVNC, }, + evetest.TestCase{ + Test: TestAppInstanceMetadata, + }, + evetest.TestCase{ + Test: TestAppUserData, + }, + evetest.TestCase{ + Test: TestAppLogs, + }, ) } diff --git a/evetest/tests/apps/userdata_test.go b/evetest/tests/apps/userdata_test.go new file mode 100644 index 00000000000..2caca88d83f --- /dev/null +++ b/evetest/tests/apps/userdata_test.go @@ -0,0 +1,268 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test delivery of application user-data (cloud-init) from the controller. + +package apps_test + +import ( + "encoding/base64" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" +) + +const ( + // Path of the file that the cloud-config user-data asks EVE to write into + // the container rootfs. + injectedFilePath = "/etc/injected_file.txt" + + // Marker env variable carried by the plain key=value user-data. + userDataMarkerKey = "EVETEST_USERDATA_MARKER" + userDataMarkerValue = "userdata-marker-value" + + // Approximate size of the plain key=value user-data blob. Large enough + // that an oversized payload would show up as a deployment failure. + userDataFillerSize = 90000 +) + +// TestAppUserData verifies both flavors of application user-data that EVE +// supports for container applications: +// +// - plain "key=value" lines, which EVE turns into environment variables of +// the container's init process, and +// - a "#cloud-config" document, of which EVE applies the write_files +// section by materializing the files inside the container rootfs. +// +// It also verifies that cloud-init is applied exactly once per user-data +// version: a file written by write_files and then modified from inside the +// application must survive an application restart untouched. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- user-data delivery is not network +// topology dependent; a single mgmt+apps port with DHCP is enough to run +// the app and reach it over SSH. +// +// Note on encoding: ApplicationInstanceConfig.UserData must be base64-encoded. +// EVE's fetchCloudInit base64-decodes the payload unconditionally -- both when +// it arrives in plaintext and when it arrives object-encrypted -- and fails app +// activation with "base64 decode failed" otherwise. This is not documented on +// the eve-api userData field, hence encodeUserData below. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps). +// - Local NI "local-ni" (10.11.12.0/24, gateway .1) on ethernet0. +// - Container app "userdata-app" (lfedge/evetest-ubuntu-ctr) with a single +// VIF on the NI, port-fwd 2222->22 and an allow-all ACL. The app is +// deployed twice - user-data cannot be modified in place, so phase 2 +// deletes the instance and deploys a new one. +// +// Phases / assertions +// ------------------- +// 1. env-userdata-app-running: deploy the app with ~90 KB of plain +// "key=value" user-data (bulk filler lines plus one marker pair). Assert +// the app still reaches RUNNING - an oversized or malformed blob must not +// wedge deployment - and then read +// /proc/1/environ inside the container and assert both the filler +// variable and the marker variable are present. /proc/1 is used rather +// than the SSH session environment because sshd sanitizes the environment +// it hands to login sessions. +// 2. env-userdata-app-deleted: delete the app and wait for +// ZSwState_INVALID. +// 3. cloudinit-app-running -> injected-file-written: deploy the app again, +// this time with a "#cloud-config" user-data whose write_files section +// writes "before_restart" to /etc/injected_file.txt with mode 0644. +// Assert the file exists inside the container with that content. +// 4. injected-file-modified -> app-restarted: overwrite the file from inside +// the app with "after_restart", then restart the app instance +// (RebootApplication, i.e. the controller's restart counter) and wait for +// it to come back to RUNNING. +// 5. Assert the file still reads "after_restart": the user-data version did +// not change, so EVE must not re-apply write_files and must not revert +// the application's own modification. +// 6. Delete the app and wait for ZSwState_INVALID. +// +// Test params +// ----------- +// - HYPERVISOR. Under Kubevirt the test waits for the cluster node to +// become ready before deploying the app. +// +// Suite placement +// --------------- +// - TestApplicationSuite (deploys an app, hence hypervisor-parameterized). +func TestAppUserData(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + // Define configurable parameters available for the test. + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + + // Get parameter values set for this test execution. + hypervisor := evetest.GetHypervisorParameterValue() + + // Set up the test harness and specify the test prerequisites. + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + // Build and apply the device configuration. + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := addLocalNI(devConfig) + + const ( + appName = "userdata-app" + sshTimeout = 20 * time.Second + appRestartTimeout = 5 * time.Minute + timeoutExcludingDownload = 10 * time.Minute + ) + + appConfig := evetest.ApplicationInstanceConfig{ + DisplayName: appName, + Activate: true, + Image: evetest.DockerContainer{ + ImageName: ubuntuCtrImage, + Tag: ubuntuCtrTag, + }, + VirtualizationMode: eveconfig.VmMode_HVM, // PV does not work in xen + CPUs: 1, + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: singleVIFWithSSH(niUUID), + UserData: envUserData(), + } + appUUID := devConfig.AddApplication(appConfig) + device.ApplyConfig(devConfig, true, true) + if hypervisor == evetest.HypervisorKubevirt { + device.WaitForClusterNodeIsReady(20 * time.Minute) + } + evetest.Checkpoint("env-userdata-config-applied") + + // Phase 1: plain key=value user-data becomes container environment. + device.WaitUntilAppIsRunning(appUUID, timeoutExcludingDownload) + evetest.Checkpoint("env-userdata-app-running") + + waitForAppSSH(t, device, appUUID) + environ, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "tr '\\0' '\\n' < /proc/1/environ", sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(strings.Split(environ, "\n")).To(ContainElement("variable=value")) + t.Expect(strings.Split(environ, "\n")).To( + ContainElement(userDataMarkerKey + "=" + userDataMarkerValue)) + + // Phase 2: user-data cannot be changed in place, so redeploy the app. + deleteAppAndWait(t, device, devConfig, appUUID) + evetest.Checkpoint("env-userdata-app-deleted") + + // Phase 3: cloud-config write_files. + appConfig.UserData = cloudInitUserData("before_restart") + appUUID = devConfig.AddApplication(appConfig) + device.ApplyConfig(devConfig, false, false) + device.WaitUntilAppIsRunning(appUUID, timeoutExcludingDownload) + evetest.Checkpoint("cloudinit-app-running") + + waitForAppSSH(t, device, appUUID) + t.Expect(readInjectedFile(t, device, appUUID, sshTimeout)).To(Equal("before_restart")) + evetest.Checkpoint("injected-file-written") + + // Phase 4: modify the injected file from inside the app, then restart it. + _, _, err = device.RunShellScriptInsideApp(appUUID, appAuth, + "echo after_restart > "+injectedFilePath, sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(readInjectedFile(t, device, appUUID, sshTimeout)).To(Equal("after_restart")) + evetest.Checkpoint("injected-file-modified") + + device.RebootApplication(appUUID, true, appRestartTimeout) + waitForAppSSH(t, device, appUUID) + evetest.Checkpoint("app-restarted") + + // Phase 5: the user-data version did not change, so cloud-init must not + // be re-applied and the app's own modification must survive. + t.Expect(readInjectedFile(t, device, appUUID, sshTimeout)).To(Equal("after_restart")) + + // Phase 6: clean up. + deleteAppAndWait(t, device, devConfig, appUUID) +} + +// encodeUserData base64-encodes the user-data payload for +// ApplicationInstanceConfig.UserData. EVE requires this: fetchCloudInit in +// domainmgr base64-decodes the payload unconditionally, on both the plaintext +// and the object-encrypted path, and fails app activation with "base64 decode +// failed" otherwise. The eve-api proto comment for userData does not mention +// it, so it is spelled out here. +func encodeUserData(userData string) string { + return base64.StdEncoding.EncodeToString([]byte(userData)) +} + +// envUserData builds a ~90 KB blob of plain "key=value" lines. EVE parses +// user-data that is not a cloud-config document as an environment variable +// map for container applications. The bulk is intentionally made of repeated +// identical pairs so that the resulting environment +// stays small while the config payload is large; a unique marker pair is +// appended to prove the whole blob was parsed, not just its beginning. +// The size refers to the decoded payload, before base64 expansion. +func envUserData() string { + const fillerLine = "variable=value\n" + marker := userDataMarkerKey + "=" + userDataMarkerValue + "\n" + var sb strings.Builder + sb.Grow(userDataFillerSize + len(marker)) + for sb.Len()+len(fillerLine) <= userDataFillerSize { + sb.WriteString(fillerLine) + } + sb.WriteString(marker) + return encodeUserData(sb.String()) +} + +// cloudInitUserData builds a cloud-config document instructing EVE to write +// the given content into injectedFilePath inside the container rootfs. +func cloudInitUserData(content string) string { + return encodeUserData(`#cloud-config +write_files: + - path: ` + injectedFilePath + ` + owner: root:root + permissions: '0644' + content: ` + content + ` +`) +} + +// readInjectedFile returns the trimmed content of the cloud-init injected +// file as seen from inside the application. +func readInjectedFile(t *WithT, device *evetest.EdgeDevice, + appUUID uuid.UUID, sshTimeout time.Duration) string { + output, _, err := device.RunShellScriptInsideApp(appUUID, appAuth, + "cat "+injectedFilePath, sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + return strings.TrimSpace(output) +} diff --git a/evetest/tests/telemetry/helpers_test.go b/evetest/tests/telemetry/helpers_test.go new file mode 100644 index 00000000000..3b703305137 --- /dev/null +++ b/evetest/tests/telemetry/helpers_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package telemetry_test + +import ( + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" +) + +const ( + // Logical name of the (single) edge device used by every test in this package. + devName = "edge-dev" + + // The single management port configured by every test in this package. + // portLogicalLabel is what the controller calls it and is therefore what + // EVE echoes back in info and metric messages; portIfName is the name of + // the underlying Linux interface. + portLogicalLabel = "ethernet0" + portIfName = "eth0" +) + +// singleMgmtPortConfig builds the device configuration shared by the tests in +// this package: one DHCP-configured management+apps port. No network instance +// and no application - these tests are only about what EVE reports about +// itself. +func singleMgmtPortConfig() *evetest.EdgeDeviceConfig { + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: portLogicalLabel, + PhysicalLabel: portIfName, + InterfaceName: portIfName, + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + return devConfig +} diff --git a/evetest/tests/telemetry/info_test.go b/evetest/tests/telemetry/info_test.go new file mode 100644 index 00000000000..2e626d05e87 --- /dev/null +++ b/evetest/tests/telemetry/info_test.go @@ -0,0 +1,173 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test the device info (ZInfoDevice) that EVE reports to the controller. + +package telemetry_test + +import ( + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// TestDeviceInfo verifies the ZInfoDevice message EVE publishes to the +// controller: EVE must report the network configuration it actually applied, +// a plausible hardware inventory, and the state of its hardware security +// module. +// +// Since the test owns the device configuration, the reported port is asserted +// exactly rather than by pattern, and the surrounding hardware fields are +// checked in the same message. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+apps port with DHCP. A single +// port keeps the expected DevicePortStatus unambiguous. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps), logical label "ethernet0". +// Nothing else: this test is about what EVE reports, not about what it +// runs. +// +// Phases / assertions +// ------------------- +// +// 1. setup-done -> config-applied. +// +// 2. port-info-reported: wait until the device info reports the +// controller-pushed DPC as current and healthy -- +// SystemAdapter.CurrentIndex==0, a single DevicePortStatus keyed +// "zedagent" (the key identifies the config source; "zedagent" means the +// controller's config won over bootstrap/last-resort), empty +// DPC.LastError, and a single port that has an IP address and no error +// recorded. These health conditions are part of the Eventually predicate +// rather than assertions after it, so a transient error early in the DPC +// verification does not flake the test. +// Then assert the port itself: +// - Ifname=eth0, Name=ethernet0 (the logical label round-trips), +// - IsMgmt=true, Up=true, +// - at least one IP address and one default router from DHCP, +// - the DPC has a LastSucceeded timestamp, i.e. EVE verified controller +// connectivity over it. +// +// Note that DevicePort.Err is never nil: zedagent encodes the port test +// results into it unconditionally and on success it carries only the +// LastSucceeded timestamp, so "no error" means an empty Description. +// +// 3. Hardware inventory in the same message is plausible: MachineArch and +// CpuArch non-empty, Ncpu >= the requested minimum, Memory and Storage +// non-zero, HostName set, BootTime set and in the past. +// +// 4. HSMStatus matches the TPM parameter: ENABLED when the device was +// created with an emulated TPM, and anything but ENABLED when it was not. +// +// Test params +// ----------- +// - TPM. Drives both whether the device VM gets an emulated TPM and the +// expected HSMStatus. +// +// Suite placement +// --------------- +// - TestTelemetrySuite. No application is deployed, so the hypervisor is +// hardcoded to KVM like the other non-app suites. +func TestDeviceInfo(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + // Define configurable parameters available for the test. + evetest.DefineTestParameters( + evetest.TPMParameter(), + ) + + // Get parameter values set for this test execution. + useTPM := evetest.GetTPMParameterValue() + + // Set up the test harness and specify the test prerequisites. + const minCPUs = 4 + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + MinCPUs: minCPUs, + WithHypervisor: evetest.HypervisorKVM, + WithTPM: useTPM, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + // Build and apply the device configuration. + devConfig := singleMgmtPortConfig() + devUpdates, stopDevWatch := device.WatchDeviceInfo() + defer stopDevWatch() + device.ApplyConfig(devConfig, true, true) + evetest.Checkpoint("config-applied") + + // Phase 2: EVE reports the network configuration it applied. + timeout := 5 * time.Minute + var devInfo *eveinfo.ZInfoDevice + t.Eventually(devUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "Device reports the controller-pushed port configuration as current", + func(info *eveinfo.ZInfoDevice) bool { + devInfo = info + sa := info.GetSystemAdapter() + if sa == nil || sa.GetCurrentIndex() != 0 || len(sa.GetStatus()) != 1 { + return false + } + dpc := sa.GetStatus()[0] + if dpc.GetKey() != "zedagent" || dpc.GetLastError() != "" || + len(dpc.GetPorts()) != 1 { + return false + } + // Note: DevicePort.Err is never nil - zedagent encodes the port + // test results into it unconditionally, and on success it carries + // just the LastSucceeded timestamp. An empty description means + // neither an error nor a warning was recorded. + port := dpc.GetPorts()[0] + return len(port.GetIPAddrs()) > 0 && port.GetErr().GetDescription() == "" + }))) + + dpc := devInfo.GetSystemAdapter().GetStatus()[0] + t.Expect(dpc.GetLastSucceeded().IsValid()).To(BeTrue()) + + port := dpc.GetPorts()[0] + t.Expect(port.GetIfname()).To(Equal(portIfName)) + t.Expect(port.GetName()).To(Equal(portLogicalLabel)) + t.Expect(port.GetIsMgmt()).To(BeTrue()) + t.Expect(port.GetUp()).To(BeTrue()) + t.Expect(port.GetIPAddrs()).ToNot(BeEmpty()) + t.Expect(port.GetDefaultRouters()).ToNot(BeEmpty()) + evetest.Checkpoint("port-info-reported") + + // Phase 3: the hardware inventory reported alongside it is plausible. + t.Expect(devInfo.GetMachineArch()).ToNot(BeEmpty()) + t.Expect(devInfo.GetCpuArch()).ToNot(BeEmpty()) + t.Expect(devInfo.GetNcpu()).To(BeNumerically(">=", minCPUs)) + t.Expect(devInfo.GetMemory()).To(BeNumerically(">", 0)) + t.Expect(devInfo.GetStorage()).To(BeNumerically(">", 0)) + t.Expect(devInfo.GetHostName()).ToNot(BeEmpty()) + t.Expect(devInfo.GetBootTime().IsValid()).To(BeTrue()) + t.Expect(devInfo.GetBootTime().AsTime()).To(BeTemporally("<", time.Now())) + + // Phase 4: the reported HSM state matches how the device was created. + if useTPM { + t.Expect(devInfo.GetHSMStatus()).To( + Equal(eveinfo.HwSecurityModuleStatus_ENABLED)) + } else { + t.Expect(devInfo.GetHSMStatus()).ToNot( + Equal(eveinfo.HwSecurityModuleStatus_ENABLED)) + } +} diff --git a/evetest/tests/telemetry/logs_test.go b/evetest/tests/telemetry/logs_test.go new file mode 100644 index 00000000000..e991809d487 --- /dev/null +++ b/evetest/tests/telemetry/logs_test.go @@ -0,0 +1,157 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test the device logs that EVE collects and uploads to the controller. + +package telemetry_test + +import ( + "fmt" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// Prefix of the marker written to /dev/kmsg. A per-run nonce is appended so +// the assertion cannot be satisfied by a log entry from an earlier run. +const kmsgMarkerPrefix = "evetest-device-log-marker" + +// TestDeviceLogs verifies that EVE collects logs and uploads them to the +// controller: a message emitted on the device in response to an externally +// triggered event must become visible to the controller, and the log stream +// of EVE's own microservices must be flowing as well. +// +// The trigger is a marker written to /dev/kmsg rather than something logged by +// a third-party daemon. Driving SSH sessions and matching on sshd's +// "Disconnected from" was tried first and does not work: the sessions are +// established (so sshd definitely handled them), but no matching entry ever +// reaches the controller within 10 minutes. /dev/kmsg is an ingestion path EVE +// documents and relies on itself -- ssh-service.sh writes there with the +// comment "this is picked up by newlogd" -- which makes the trigger +// deterministic and the assertion immune to third-party log wording. +// +// Two ingestion paths are covered: +// - /dev/kmsg -> getKernelMsg -> newlogd (source "kernel"), driven by the +// test, and +// - the pillar agents' memlog path (source = agent name), which is EVE +// logging on its own. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- log upload only needs working +// controller connectivity over a single mgmt port. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps), logical label "ethernet0". +// - debug.kernel.loglevel and debug.kernel.remote.loglevel set to "info". +// Both already default to "info" (types.SyslogKernelDefaultLogLevel) and +// a userspace write to /dev/kmsg lands at that priority, but the kernel +// path is gated by its own knobs -- separate from the debug.default.* +// ones the framework sets -- so the test states its dependency instead of +// inheriting it. +// - The framework defaults already enable newlog.allow.fastupload. +// +// Phases / assertions +// ------------------- +// 1. setup-done -> config-applied. +// 2. Subscribe to device logs matching a per-run marker string *before* +// emitting it, then write the marker to /dev/kmsg over SSH. +// 3. kernel-log-received: the marker reaches the controller. Assert +// Source="kernel" (the entry was attributed to the right ingestion +// path), Severity is populated, and the timestamp is not in the future -- +// i.e. a well-formed log record, not merely a matching string. +// 4. The pillar agents' log stream is flowing too: entries with +// Source="zedagent" have been uploaded (base/logobjecttypes.go sets the +// "source" field to the agent name). +// +// Timing note: logs are batched into gzip bundles before upload, so the wait +// in phase 3 is generous even with fast upload enabled. +// +// Test params +// ----------- +// - TPM. Declared only so that every test in the suite states the same +// device requirements and the framework can reuse one VM; nothing here +// depends on the TPM. +// +// Suite placement +// --------------- +// - TestTelemetrySuite. No application is deployed, so the hypervisor is +// hardcoded to KVM like the other non-app suites. +func TestDeviceLogs(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + // Define configurable parameters available for the test. + evetest.DefineTestParameters( + evetest.TPMParameter(), + ) + + // Get parameter values set for this test execution. + useTPM := evetest.GetTPMParameterValue() + + // Set up the test harness and specify the test prerequisites. + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: evetest.HypervisorKVM, + WithTPM: useTPM, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + // Build and apply the device configuration. The kernel log path has its + // own level knobs, independent of the framework-wide debug.default.*. + devConfig := singleMgmtPortConfig() + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueString(pillartypes.KernelLogLevel, "info") + cfgProps.SetGlobalValueString(pillartypes.KernelRemoteLogLevel, "info") + devConfig.SetConfigProperties(cfgProps) + device.ApplyConfig(devConfig, true, true) + evetest.Checkpoint("config-applied") + + // Phase 2: subscribe first, then emit the marker. + marker := fmt.Sprintf("%s-%d", kmsgMarkerPrefix, time.Now().UnixNano()) + logs, stopLogWatch := device.WatchLogs(evetest.LogMsgMatch{ + MsgHasSubstring: marker, + }) + defer stopLogWatch() + + const ( + sshTimeout = 30 * time.Second + logTimeout = 10 * time.Minute + ) + _, _, err := device.RunShellScript( + fmt.Sprintf("echo %q > /dev/kmsg", marker), sshTimeout, 0) + t.Expect(err).ToNot(HaveOccurred()) + evetest.Logger().Infof( + "Emitted marker %q on the device, waiting for it to reach the controller...", + marker) + + // Phase 3: the message reaches the controller. + var logMsg evetest.LogMsg + t.Eventually(logs, logTimeout).Should(Receive(&logMsg), + "device log marker was not uploaded to the controller") + t.Expect(logMsg.Message).To(ContainSubstring(marker)) + t.Expect(logMsg.Source).To(Equal("kernel")) + t.Expect(logMsg.Severity).ToNot(BeEmpty()) + t.Expect(logMsg.Timestamp).To(BeTemporally("<", time.Now())) + evetest.Checkpoint("kernel-log-received") + + // Phase 4: EVE's own microservices are logging to the controller as well. + agentLogs := device.GetLogs(evetest.LogMsgMatch{Source: "zedagent"}) + t.Expect(agentLogs).ToNot(BeEmpty(), + "no zedagent log messages were uploaded to the controller") +} diff --git a/evetest/tests/telemetry/metrics_test.go b/evetest/tests/telemetry/metrics_test.go new file mode 100644 index 00000000000..dd50efbe07f --- /dev/null +++ b/evetest/tests/telemetry/metrics_test.go @@ -0,0 +1,164 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test the device metrics (DeviceMetric) that EVE reports to the controller. + +package telemetry_test + +import ( + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + evemetrics "github.com/lf-edge/eve-api/go/metrics" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// TestDeviceMetrics verifies the DeviceMetric message EVE publishes to the +// controller: per-port network counters attributed to the right port, plus +// the memory, CPU and controller-connectivity counters that ride along in the +// same message. +// +// Note that NetworkMetric.iName is the *logical label* from the controller +// config, not the Linux interface name; localName carries the latter. The two +// are asserted separately here, since a device model that happens to label its +// port "eth0" would otherwise hide a mix-up between them. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- one mgmt+apps port with DHCP. Traffic to +// the controller flows over it continuously, so its counters are +// guaranteed to be non-zero without the test generating any load. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps), logical label "ethernet0". +// The framework default already lowers the metric publishing interval to +// 20s, so no extra tuning is needed. +// +// Phases / assertions +// ------------------- +// 1. setup-done -> config-applied. +// 2. network-metrics-reported: wait for a DeviceMetric carrying a +// NetworkMetric whose iName is the port's logical label "ethernet0", with +// non-zero TxBytes, RxBytes, TxPkts and RxPkts -- the device is talking to +// the controller over this port, so counters must be moving -- and +// carrying non-zero DeviceMemory.MemoryMB and CpuMetric.TotalNs. Those +// last two come from a different producer (zedagent fills them only once +// domainmgr has published the host DomainMetric), so they are part of the +// wait rather than assertions on whichever message first satisfied the +// network condition. Then assert localName is the interface name eth0. +// 3. Memory metrics in the same message: UsedEveMB is non-zero and does not +// exceed the reported total device memory. +// 4. CPU metrics: CpuMetric.UpTime is set and in the past. +// 5. Controller connectivity: at least one ZedcloudMetric with Success > 0 +// and a LastSuccess timestamp -- EVE accounts for the API calls it makes, +// which is the metric the controller uses to judge device liveness. +// +// All assertions are made against a single message captured by the +// Eventually, so the values are mutually consistent rather than sampled from +// different publishing rounds. +// +// Test params +// ----------- +// - TPM. Declared only so that every test in the suite states the same +// device requirements and the framework can reuse one VM; nothing here +// depends on the TPM. +// +// Suite placement +// --------------- +// - TestTelemetrySuite. No application is deployed, so the hypervisor is +// hardcoded to KVM like the other non-app suites. +func TestDeviceMetrics(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + // Define configurable parameters available for the test. + evetest.DefineTestParameters( + evetest.TPMParameter(), + ) + + // Get parameter values set for this test execution. + useTPM := evetest.GetTPMParameterValue() + + // Set up the test harness and specify the test prerequisites. + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + WithHypervisor: evetest.HypervisorKVM, + WithTPM: useTPM, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + // Build and apply the device configuration. + devConfig := singleMgmtPortConfig() + metricUpdates, stopMetricWatch := device.WatchDeviceMetrics() + defer stopMetricWatch() + device.ApplyConfig(devConfig, true, true) + evetest.Checkpoint("config-applied") + + // Phase 2: per-port network counters are reported and moving. + timeout := 5 * time.Minute + var metrics *evemetrics.DeviceMetric + var portMetric *evemetrics.NetworkMetric + t.Eventually(metricUpdates, timeout).Should(Receive(matchers.SatisfyPredicate( + "Device reports non-zero network, memory and CPU counters", + func(dm *evemetrics.DeviceMetric) bool { + metrics = dm + portMetric = nil + for _, nm := range dm.GetNetwork() { + if nm.GetIName() == portLogicalLabel { + portMetric = nm + break + } + } + if portMetric == nil || + portMetric.GetTxBytes() == 0 || portMetric.GetRxBytes() == 0 || + portMetric.GetTxPkts() == 0 || portMetric.GetRxPkts() == 0 { + return false + } + // DeviceMemory and CpuMetric.TotalNs come from a different + // producer than the network counters: zedagent only fills them in + // once domainmgr has published the host DomainMetric (see + // handlemetrics.go, the lookupDomainMetric(nilUUID) branch). Wait + // for a message carrying all of them rather than asserting them + // on whichever message happened to satisfy the network condition. + return dm.GetDeviceMemory().GetMemoryMB() > 0 && + dm.GetCpuMetric().GetTotalNs() > 0 + }))) + t.Expect(portMetric.GetLocalName()).To(Equal(portIfName)) + evetest.Checkpoint("network-metrics-reported") + + // Phase 3: memory metrics. + deviceMemory := metrics.GetDeviceMemory() + t.Expect(deviceMemory.GetUsedEveMB()).To(BeNumerically(">", 0)) + t.Expect(deviceMemory.GetUsedEveMB()).To( + BeNumerically("<=", deviceMemory.GetMemoryMB())) + + // Phase 4: CPU metrics. + cpuMetric := metrics.GetCpuMetric() + t.Expect(cpuMetric.GetUpTime().IsValid()).To(BeTrue()) + t.Expect(cpuMetric.GetUpTime().AsTime()).To(BeTemporally("<", time.Now())) + + // Phase 5: controller connectivity is accounted for. + var successfulSends uint64 + for _, zm := range metrics.GetZedcloud() { + if zm.GetSuccess() > 0 { + successfulSends += zm.GetSuccess() + t.Expect(zm.GetLastSuccess().IsValid()).To(BeTrue()) + } + } + t.Expect(successfulSends).To(BeNumerically(">", 0), + "EVE must account for the successful API calls it made to the controller") +} diff --git a/evetest/tests/telemetry/testsuite_test.go b/evetest/tests/telemetry/testsuite_test.go new file mode 100644 index 00000000000..55a48269da1 --- /dev/null +++ b/evetest/tests/telemetry/testsuite_test.go @@ -0,0 +1,51 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package telemetry_test + +import ( + "testing" + + "github.com/lf-edge/eve/evetest" +) + +// TestTelemetrySuite drives the three channels through which EVE reports +// itself to the controller: info messages, metric messages and logs. None of +// the subtests deploys an application -- they exercise the EVE control plane +// only -- and therefore none parameterizes the hypervisor; all hardcode +// HypervisorKVM, as the other non-application suites do. +// +// Every subtest declares the TPM parameter and passes it to +// RequireEdgeDevice, so all three state identical device requirements and the +// framework reuses a single VM across the suite. Only TestDeviceInfo actually +// asserts on the TPM (via HSMStatus); for the other two the parameter merely +// selects which device flavor the suite runs on. +// +// Subtests +// -------- +// - TestDeviceInfo -- ZInfoDevice reports the applied port configuration, +// a plausible hardware inventory and the HSM state. +// - TestDeviceMetrics -- DeviceMetric reports moving per-port network +// counters plus memory, CPU and controller-connectivity counters. +// - TestDeviceLogs -- logs produced by EVE services reach the controller. +func TestTelemetrySuite(test *testing.T) { + evetest.Init(test) + defer evetest.Close() + + // Define parameters for the entire test suite. + evetest.DefineTestParameters( + evetest.TPMParameter(), + ) + + evetest.RunTestSuite( + evetest.TestCase{ + Test: TestDeviceInfo, + }, + evetest.TestCase{ + Test: TestDeviceMetrics, + }, + evetest.TestCase{ + Test: TestDeviceLogs, + }, + ) +}