From 70d01bf35dff1b5c73c1f2c7b8ed194304a9608d Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:00:00 +0000 Subject: [PATCH 01/15] cputopology: discover the CPU topology from sysfs Adds a standalone package that reads the machine's socket, physical-core, SMT-sibling, NUMA and L3 structure straight from sysfs, in pure Go with no CGO and no external topology library. Device classes EVE targets often ship neither, and the native dependency previously considered for this proved fragile on client and non-server SKUs. SMT siblings are identified by a shared (socket, core_id) key, which is authoritative on every architecture we target. Grouping by a cache id would be wrong: on Intel hybrid parts an efficiency-core module exposes one shared L2 across four distinct physical cores with no SMT, which such a key would model as a single four-thread core. Discovery degrades to a flat model rather than failing when sysfs cannot be read, so a caller always has a usable topology and simply loses the locality guarantees it cannot substantiate. The package deliberately depends on nothing else in pillar so the allocator, the hardware inventory and a future cluster-side consumer can share it. Signed-off-by: Mikhail Malyshev --- pkg/pillar/cputopology/sysfs.go | 303 +++++++++++++ pkg/pillar/cputopology/sysfs_test.go | 560 ++++++++++++++++++++++++ pkg/pillar/cputopology/topology.go | 123 ++++++ pkg/pillar/cputopology/topology_test.go | 77 ++++ 4 files changed, 1063 insertions(+) create mode 100644 pkg/pillar/cputopology/sysfs.go create mode 100644 pkg/pillar/cputopology/sysfs_test.go create mode 100644 pkg/pillar/cputopology/topology.go create mode 100644 pkg/pillar/cputopology/topology_test.go diff --git a/pkg/pillar/cputopology/sysfs.go b/pkg/pillar/cputopology/sysfs.go new file mode 100644 index 00000000000..01fd613ab7c --- /dev/null +++ b/pkg/pillar/cputopology/sysfs.go @@ -0,0 +1,303 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cputopology + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" +) + +// defaultSysfsRoot is the standard Linux sysfs location for CPU and NUMA +// node topology information. +var defaultSysfsRoot = "/sys/devices/system" + +// ErrNoCPUTopology reports that sysfs exposed no usable CPU topology at all +// (an empty "online" file, or no cpuN directories), as opposed to exposing +// something that could not be read. +var ErrNoCPUTopology = errors.New("sysfs exposed no usable CPU topology") + +// DiscoverTopology reads CPU topology from sysfs. The returned *Topology is +// always non-nil: on failure it degrades to a flat single-thread-per-core +// model so reporting paths keep working. A non-nil error therefore means the +// topology is degraded (Topology.Degraded set), not absent, and callers doing +// CPU placement must refuse it rather than fall back to it. +func DiscoverTopology() (*Topology, error) { + infos, err := readSysfsCoreInfos(defaultSysfsRoot) + if err != nil { + return flatTopology(runtime.NumCPU()), err + } + if len(infos) == 0 { + return flatTopology(runtime.NumCPU()), ErrNoCPUTopology + } + return BuildTopology(infos), nil +} + +// flatTopology builds a degraded topology of n single-thread physical +// cores, all on socket 0 / NUMA node 0, with unknown L3 ids. Used when sysfs +// discovery fails. The CPU ids it invents (0..n-1) need not be real host CPU +// ids, hence Topology.Degraded - see its documentation. +func flatTopology(n int) *Topology { + if n < 1 { + n = 1 + } + infos := make([]CoreInfo, n) + for i := 0; i < n; i++ { + // Nothing is known about the cache hierarchy here, and saying "L3 domain + // 0" would claim every CPU shares one cache. + infos[i] = CoreInfo{LCore: uint(i), Socket: 0, CoreID: uint(i), NUMA: 0, + L3Unknown: true} + } + topo := BuildTopology(infos) + topo.Degraded = true + return topo +} + +// readSysfsCoreInfos reads per-logical-CPU topology coordinates from a +// sysfs tree rooted at root (normally /sys/devices/system; tests inject a +// temp dir with the same layout). +func readSysfsCoreInfos(root string) ([]CoreInfo, error) { + cpuRoot := filepath.Join(root, "cpu") + + online, err := readOnlineCPUs(cpuRoot) + if err != nil { + return nil, err + } + + nodeOfCPU, numaExposed, err := readNUMAMapping(root) + if err != nil { + return nil, err + } + + infos := make([]CoreInfo, 0, len(online)) + for _, cpu := range online { + ci := CoreInfo{LCore: cpu} + + cpuDir := filepath.Join(cpuRoot, fmt.Sprintf("cpu%d", cpu)) + + if v, ok := readOptionalUint(filepath.Join(cpuDir, "topology", "physical_package_id")); ok { + ci.Socket = v + } else { + ci.Socket = 0 + } + + coreID, err := readRequiredUint(filepath.Join(cpuDir, "topology", "core_id")) + if err != nil { + return nil, fmt.Errorf("cpu%d: missing core_id: %w", cpu, err) + } + ci.CoreID = coreID + + var l3Known bool + ci.L3ID, l3Known = readL3ID(filepath.Join(cpuDir, "cache")) + ci.L3Unknown = !l3Known + + // A CPU missing from the node listings while the kernel does expose + // NUMA information is a hole in the model, not a hint that it lives on + // node 0: guessing would let a single-NUMA-node placement request be + // satisfied with cores from two sockets and reported as optimal. + if numa, ok := nodeOfCPU[cpu]; ok { + ci.NUMA = numa + } else if numaExposed { + return nil, fmt.Errorf("cpu%d: no NUMA node covers it in %s", + cpu, filepath.Join(root, "node")) + } + + infos = append(infos, ci) + } + + return infos, nil +} + +// readOnlineCPUs returns the online logical CPU ids in ascending order. +func readOnlineCPUs(cpuRoot string) ([]uint, error) { + cpus, err := readOnlineCPUsUnordered(cpuRoot) + if err != nil { + return nil, err + } + // Ascending order is an enforced postcondition, not an observation about + // the kernel's output: SMT sibling selection picks Siblings[0] as the + // thread to run on, so it must not depend on file or directory ordering. + slices.Sort(cpus) + return cpus, nil +} + +// readOnlineCPUsUnordered reads /online (a Linux cpu range list, +// e.g. "0-7,16") and falls back to enumerating /cpuN directories if +// that file is missing. +func readOnlineCPUsUnordered(cpuRoot string) ([]uint, error) { + onlinePath := filepath.Join(cpuRoot, "online") + data, err := os.ReadFile(onlinePath) + if err == nil { + return parseCPURangeList(string(data)) + } + if !os.IsNotExist(err) { + return nil, fmt.Errorf("reading %s: %w", onlinePath, err) + } + + entries, err := os.ReadDir(cpuRoot) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", cpuRoot, err) + } + var cpus []uint + for _, e := range entries { + if !e.IsDir() { + continue + } + name := e.Name() + if !strings.HasPrefix(name, "cpu") { + continue + } + n, err := strconv.ParseUint(strings.TrimPrefix(name, "cpu"), 10, 32) + if err != nil { + continue + } + cpus = append(cpus, uint(n)) + } + return cpus, nil +} + +// readNUMAMapping builds a map from logical CPU id to NUMA node id by +// reading /node/node*/cpulist. exposed is false when /node does +// not exist at all, i.e. the kernel publishes no NUMA information; the caller +// then treats the whole machine as node 0, which is the truth on such a +// system. A node directory that does exist but whose cpulist cannot be read +// or parsed is an error instead: quietly defaulting those CPUs to node 0 +// would fabricate NUMA locality that a placement decision then trusts. +func readNUMAMapping(root string) (mapping map[uint]uint, exposed bool, err error) { + nodeRoot := filepath.Join(root, "node") + entries, err := os.ReadDir(nodeRoot) + if err != nil { + if os.IsNotExist(err) { + return map[uint]uint{}, false, nil + } + return nil, false, fmt.Errorf("reading %s: %w", nodeRoot, err) + } + + mapping = map[uint]uint{} + for _, e := range entries { + name := e.Name() + if !e.IsDir() || !strings.HasPrefix(name, "node") { + continue + } + nodeID, err := strconv.ParseUint(strings.TrimPrefix(name, "node"), 10, 32) + if err != nil { + continue + } + cpulistPath := filepath.Join(nodeRoot, name, "cpulist") + data, err := os.ReadFile(cpulistPath) + if err != nil { + return nil, true, fmt.Errorf("reading %s: %w", cpulistPath, err) + } + // An empty cpulist is legitimate: a memory-only node has no CPUs. + cpus, err := parseCPURangeList(string(data)) + if err != nil { + return nil, true, fmt.Errorf("parsing %s: %w", cpulistPath, err) + } + for _, cpu := range cpus { + mapping[cpu] = uint(nodeID) + } + } + return mapping, true, nil +} + +// readL3ID finds the cache index under cacheDir whose level file contains +// "3" and returns its id. ok is false when the platform exposes no such id - +// no readable cache directory, no level-3 index, or an index without an "id" +// file (common on ARM64). An unknown id must stay distinguishable from a real +// id 0, otherwise every core appears to share one L3 domain and a +// cache-splitting placement is never noticed. +func readL3ID(cacheDir string) (uint, bool) { + entries, err := os.ReadDir(cacheDir) + if err != nil { + return 0, false + } + for _, e := range entries { + name := e.Name() + if !e.IsDir() || !strings.HasPrefix(name, "index") { + continue + } + levelPath := filepath.Join(cacheDir, name, "level") + level, ok := readOptionalUint(levelPath) + if !ok || level != 3 { + continue + } + return readOptionalUint(filepath.Join(cacheDir, name, "id")) + } + return 0, false +} + +// readOptionalUint reads a single unsigned integer from path, treating +// any error (missing file, parse failure) or a negative value (e.g. "-1" +// for physical_package_id on single-socket systems) as "not present". +func readOptionalUint(path string) (uint, bool) { + data, err := os.ReadFile(path) + if err != nil { + return 0, false + } + s := strings.TrimSpace(string(data)) + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || n < 0 { + return 0, false + } + return uint(n), true +} + +// readRequiredUint reads a single unsigned integer from path, returning +// an error if the file is missing, unparsable, or negative. +func readRequiredUint(path string) (uint, error) { + data, err := os.ReadFile(path) + if err != nil { + return 0, err + } + s := strings.TrimSpace(string(data)) + n, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0, fmt.Errorf("parsing %s (%q): %w", path, s, err) + } + if n < 0 { + return 0, fmt.Errorf("parsing %s: unexpected negative value %d", path, n) + } + return uint(n), nil +} + +// parseCPURangeList parses a Linux cpu range list such as "0-3,7" into +// []uint{0,1,2,3,7}. Whitespace/newlines are trimmed before parsing. +func parseCPURangeList(s string) ([]uint, error) { + s = strings.TrimSpace(s) + if s == "" { + return nil, nil + } + var result []uint + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if idx := strings.Index(part, "-"); idx >= 0 { + lo, err := strconv.ParseUint(part[:idx], 10, 32) + if err != nil { + return nil, fmt.Errorf("parsing range %q: %w", part, err) + } + hi, err := strconv.ParseUint(part[idx+1:], 10, 32) + if err != nil { + return nil, fmt.Errorf("parsing range %q: %w", part, err) + } + for v := lo; v <= hi; v++ { + result = append(result, uint(v)) + } + } else { + v, err := strconv.ParseUint(part, 10, 32) + if err != nil { + return nil, fmt.Errorf("parsing cpu id %q: %w", part, err) + } + result = append(result, uint(v)) + } + } + return result, nil +} diff --git a/pkg/pillar/cputopology/sysfs_test.go b/pkg/pillar/cputopology/sysfs_test.go new file mode 100644 index 00000000000..829f95b58de --- /dev/null +++ b/pkg/pillar/cputopology/sysfs_test.go @@ -0,0 +1,560 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cputopology + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +// writeFile creates parent directories as needed and writes content to path. +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("MkdirAll(%s): %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile(%s): %v", path, err) + } +} + +// buildFixtureTree builds a fake sysfs tree under dir mimicking a +// 2-physical-core, SMT2, single-socket, single-NUMA box: cpu0 & cpu1 share +// core_id 0; cpu2 & cpu3 share core_id 1. All four logical CPUs share L3 +// index3/id=0 and NUMA node0. +func buildFixtureTree(t *testing.T, dir string) { + t.Helper() + + writeFile(t, filepath.Join(dir, "cpu", "online"), "0-3\n") + + coreIDs := map[int]int{0: 0, 1: 0, 2: 1, 3: 1} + for cpu, coreID := range coreIDs { + base := filepath.Join(dir, "cpu", "cpu"+itoa(cpu)) + writeFile(t, filepath.Join(base, "topology", "physical_package_id"), "0\n") + writeFile(t, filepath.Join(base, "topology", "core_id"), itoa(coreID)+"\n") + writeFile(t, filepath.Join(base, "cache", "index3", "level"), "3\n") + writeFile(t, filepath.Join(base, "cache", "index3", "id"), "0\n") + } + + writeFile(t, filepath.Join(dir, "node", "node0", "cpulist"), "0-3\n") +} + +// itoa avoids pulling in strconv just for test fixture path building. +func itoa(n int) string { + if n == 0 { + return "0" + } + digits := "" + neg := n < 0 + if neg { + n = -n + } + for n > 0 { + digits = string(rune('0'+n%10)) + digits + n /= 10 + } + if neg { + digits = "-" + digits + } + return digits +} + +func TestReadSysfsCoreInfos(t *testing.T) { + dir := t.TempDir() + buildFixtureTree(t, dir) + + infos, err := readSysfsCoreInfos(dir) + if err != nil { + t.Fatalf("readSysfsCoreInfos: %v", err) + } + + topo := BuildTopology(infos) + + if len(topo.Cores) != 2 { + t.Fatalf("expected 2 physical cores, got %d", len(topo.Cores)) + } + if topo.NumLCPUs != 4 { + t.Fatalf("expected NumLCPUs == 4, got %d", topo.NumLCPUs) + } + + var core0, core1 *PhysicalCore + for i := range topo.Cores { + switch topo.Cores[i].CoreID { + case 0: + core0 = &topo.Cores[i] + case 1: + core1 = &topo.Cores[i] + } + } + if core0 == nil || core1 == nil { + t.Fatalf("expected cores with CoreID 0 and 1, got %+v", topo.Cores) + } + if len(core0.Siblings) != 2 || core0.Siblings[0] != LCPU(0) || core0.Siblings[1] != LCPU(1) { + t.Fatalf("expected CoreID 0 siblings {0,1}, got %v", core0.Siblings) + } + if len(core1.Siblings) != 2 || core1.Siblings[0] != LCPU(2) || core1.Siblings[1] != LCPU(3) { + t.Fatalf("expected CoreID 1 siblings {2,3}, got %v", core1.Siblings) + } + + if len(topo.L3Cores[0]) != 2 { + t.Fatalf("expected L3Cores[0] to have 2 cores, got %d", len(topo.L3Cores[0])) + } + if len(topo.NUMACores[0]) != 2 { + t.Fatalf("expected NUMACores[0] to have 2 cores, got %d", len(topo.NUMACores[0])) + } + for _, ci := range infos { + if ci.L3Unknown { + t.Fatalf("cpu%d should have a known L3 id, the fixture exposes cache/index3/id", ci.LCore) + } + } + if topo.Degraded { + t.Fatalf("topology read from sysfs must not be marked degraded") + } + for i := range topo.Cores { + if topo.Cores[i].L3Unknown { + t.Fatalf("core %+v should have a known L3 id", topo.Cores[i]) + } + } +} + +// TestReadSysfsCoreInfos_NoOnlineFile verifies that when /online is +// missing, readOnlineCPUs falls back to enumerating cpu/cpuN directories and +// all CPUs are still discovered. +func TestReadSysfsCoreInfos_NoOnlineFile(t *testing.T) { + dir := t.TempDir() + buildFixtureTree(t, dir) + + // Remove the online file to force the cpuN directory enumeration + // fallback path in readOnlineCPUs. + if err := os.Remove(filepath.Join(dir, "cpu", "online")); err != nil { + t.Fatalf("Remove(online): %v", err) + } + + infos, err := readSysfsCoreInfos(dir) + if err != nil { + t.Fatalf("readSysfsCoreInfos: %v", err) + } + + if len(infos) != 4 { + t.Fatalf("expected 4 CPUs discovered via directory fallback, got %d", len(infos)) + } + seen := map[uint]bool{} + for _, ci := range infos { + seen[ci.LCore] = true + } + for cpu := uint(0); cpu < 4; cpu++ { + if !seen[cpu] { + t.Fatalf("expected cpu%d to be discovered, got infos %+v", cpu, infos) + } + } +} + +// TestReadSysfsCoreInfos_SocketDefault verifies that a missing or literal +// "-1" physical_package_id both fall back to Socket 0. +func TestReadSysfsCoreInfos_SocketDefault(t *testing.T) { + cases := []struct { + name string + writePkgIDFn func(t *testing.T, path string) + wantSocketVal uint + }{ + { + name: "missing physical_package_id file", + writePkgIDFn: func(t *testing.T, path string) { + t.Helper() + if err := os.Remove(path); err != nil { + t.Fatalf("Remove(%s): %v", path, err) + } + }, + wantSocketVal: 0, + }, + { + name: "physical_package_id is -1", + writePkgIDFn: func(t *testing.T, path string) { + t.Helper() + writeFile(t, path, "-1\n") + }, + wantSocketVal: 0, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + buildFixtureTree(t, dir) + + pkgIDPath := filepath.Join(dir, "cpu", "cpu0", "topology", "physical_package_id") + tc.writePkgIDFn(t, pkgIDPath) + + infos, err := readSysfsCoreInfos(dir) + if err != nil { + t.Fatalf("readSysfsCoreInfos: %v", err) + } + + var found bool + for _, ci := range infos { + if ci.LCore == 0 { + found = true + if ci.Socket != tc.wantSocketVal { + t.Fatalf("expected cpu0 Socket == %d, got %d", tc.wantSocketVal, ci.Socket) + } + } + } + if !found { + t.Fatalf("expected cpu0 in infos, got %+v", infos) + } + }) + } +} + +// TestReadSysfsCoreInfos_L3Unknown verifies that every way sysfs can fail to +// name a CPU's L3 cache is reported as "unknown" rather than as L3 id 0, +// which is a real id on most machines. +func TestReadSysfsCoreInfos_L3Unknown(t *testing.T) { + cases := []struct { + name string + breakFn func(t *testing.T, cacheDir string) + }{ + { + name: "no cache directory at all", + breakFn: func(t *testing.T, cacheDir string) { + t.Helper() + if err := os.RemoveAll(cacheDir); err != nil { + t.Fatalf("RemoveAll(%s): %v", cacheDir, err) + } + }, + }, + { + name: "no index with level 3", + breakFn: func(t *testing.T, cacheDir string) { + t.Helper() + if err := os.RemoveAll(cacheDir); err != nil { + t.Fatalf("RemoveAll(%s): %v", cacheDir, err) + } + writeFile(t, filepath.Join(cacheDir, "index2", "level"), "2\n") + writeFile(t, filepath.Join(cacheDir, "index2", "id"), "5\n") + }, + }, + { + // Typical on ARM64: the cache index exists but carries no id. + name: "level 3 index without an id file", + breakFn: func(t *testing.T, cacheDir string) { + t.Helper() + if err := os.Remove(filepath.Join(cacheDir, "index3", "id")); err != nil { + t.Fatalf("Remove(id): %v", err) + } + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + buildFixtureTree(t, dir) + tc.breakFn(t, filepath.Join(dir, "cpu", "cpu0", "cache")) + + infos, err := readSysfsCoreInfos(dir) + if err != nil { + t.Fatalf("readSysfsCoreInfos: %v", err) + } + + var found bool + for _, ci := range infos { + if ci.LCore != 0 { + continue + } + found = true + if !ci.L3Unknown { + t.Fatalf("expected cpu0 to report an unknown L3 id, got L3ID %d", ci.L3ID) + } + } + if !found { + t.Fatalf("expected cpu0 in infos, got %+v", infos) + } + + // The unknown must survive into the physical core: cpu0's core + // has one sibling (cpu1) that still reports an L3 id. + topo := BuildTopology(infos) + if pc := topo.ByLCPU[LCPU(0)]; pc == nil || !pc.L3Unknown { + t.Fatalf("expected cpu0's core to report an unknown L3 id, got %+v", pc) + } + }) + } +} + +// TestReadSysfsCoreInfos_NoNodeDir verifies that when the sysfs root has no +// node directory at all, every CPU falls back to NUMA node 0. +func TestReadSysfsCoreInfos_NoNodeDir(t *testing.T) { + dir := t.TempDir() + buildFixtureTree(t, dir) + + nodeDir := filepath.Join(dir, "node") + if err := os.RemoveAll(nodeDir); err != nil { + t.Fatalf("RemoveAll(%s): %v", nodeDir, err) + } + + infos, err := readSysfsCoreInfos(dir) + if err != nil { + t.Fatalf("readSysfsCoreInfos: %v", err) + } + + for _, ci := range infos { + if ci.NUMA != 0 { + t.Fatalf("expected NUMA == 0 for cpu%d when node dir is absent, got %d", ci.LCore, ci.NUMA) + } + } +} + +// TestReadSysfsCoreInfos_NUMAErrors verifies that a node directory which +// exists but does not yield a usable CPU-to-node mapping is an error. Falling +// back to node 0 would let a single-NUMA-node placement request be satisfied +// with cores from two sockets and still be reported as optimal. +func TestReadSysfsCoreInfos_NUMAErrors(t *testing.T) { + cases := []struct { + name string + breakFn func(t *testing.T, dir string) + }{ + { + name: "unparsable cpulist", + breakFn: func(t *testing.T, dir string) { + t.Helper() + writeFile(t, filepath.Join(dir, "node", "node0", "cpulist"), "zero-three\n") + }, + }, + { + name: "unreadable cpulist", + breakFn: func(t *testing.T, dir string) { + t.Helper() + if os.Geteuid() == 0 { + t.Skip("root bypasses file permissions") + } + path := filepath.Join(dir, "node", "node0", "cpulist") + if err := os.Chmod(path, 0o000); err != nil { + t.Fatalf("Chmod(%s): %v", path, err) + } + }, + }, + { + name: "online CPU covered by no node", + breakFn: func(t *testing.T, dir string) { + t.Helper() + writeFile(t, filepath.Join(dir, "node", "node0", "cpulist"), "0-2\n") + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + buildFixtureTree(t, dir) + tc.breakFn(t, dir) + + if _, err := readSysfsCoreInfos(dir); err == nil { + t.Fatalf("expected an error, got nil") + } + }) + } +} + +// TestReadSysfsCoreInfos_MissingCoreID verifies that a missing core_id file +// on an online CPU is treated as a hard error. +func TestReadSysfsCoreInfos_MissingCoreID(t *testing.T) { + dir := t.TempDir() + buildFixtureTree(t, dir) + + coreIDPath := filepath.Join(dir, "cpu", "cpu0", "topology", "core_id") + if err := os.Remove(coreIDPath); err != nil { + t.Fatalf("Remove(%s): %v", coreIDPath, err) + } + + _, err := readSysfsCoreInfos(dir) + if err == nil { + t.Fatalf("expected readSysfsCoreInfos to return an error when core_id is missing, got nil") + } +} + +// TestParseCPURangeList verifies multi-range parsing, e.g. "0-3,7" expands +// to [0 1 2 3 7]. This is exercised both directly (the helper is in-package +// and exported to the test via the shared package) and indirectly through +// a cpu/online fixture. +func TestParseCPURangeList(t *testing.T) { + got, err := parseCPURangeList("0-3,7") + if err != nil { + t.Fatalf("parseCPURangeList: %v", err) + } + want := []uint{0, 1, 2, 3, 7} + if len(got) != len(want) { + t.Fatalf("expected %v, got %v", want, got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("expected %v, got %v", want, got) + } + } +} + +// TestReadSysfsCoreInfos_MultiRangeOnline exercises the "0-3,7" range list +// via the cpu/online fixture file, confirming readOnlineCPUs (and therefore +// parseCPURangeList) is applied correctly end-to-end. +func TestReadSysfsCoreInfos_MultiRangeOnline(t *testing.T) { + dir := t.TempDir() + buildFixtureTree(t, dir) + + writeFile(t, filepath.Join(dir, "cpu", "online"), "0-3,7\n") + // cpu7 needs its own topology/cache files and a NUMA node covering it + // since buildFixtureTree only wires up cpu0-cpu3. + writeFile(t, filepath.Join(dir, "node", "node0", "cpulist"), "0-3,7\n") + base := filepath.Join(dir, "cpu", "cpu7") + writeFile(t, filepath.Join(base, "topology", "physical_package_id"), "0\n") + writeFile(t, filepath.Join(base, "topology", "core_id"), "2\n") + writeFile(t, filepath.Join(base, "cache", "index3", "level"), "3\n") + writeFile(t, filepath.Join(base, "cache", "index3", "id"), "0\n") + + infos, err := readSysfsCoreInfos(dir) + if err != nil { + t.Fatalf("readSysfsCoreInfos: %v", err) + } + + got := make([]uint, 0, len(infos)) + for _, ci := range infos { + got = append(got, ci.LCore) + } + want := []uint{0, 1, 2, 3, 7} + if len(got) != len(want) { + t.Fatalf("expected LCores %v, got %v", want, got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("expected LCores %v, got %v", want, got) + } + } +} + +// TestReadOnlineCPUs_Sorted verifies the ascending-order postcondition even +// when the online file lists ranges out of order: sibling selection uses +// Siblings[0] and must not inherit the file's ordering. +func TestReadOnlineCPUs_Sorted(t *testing.T) { + dir := t.TempDir() + buildFixtureTree(t, dir) + writeFile(t, filepath.Join(dir, "cpu", "online"), "2-3,0-1\n") + + cpus, err := readOnlineCPUs(filepath.Join(dir, "cpu")) + if err != nil { + t.Fatalf("readOnlineCPUs: %v", err) + } + want := []uint{0, 1, 2, 3} + if len(cpus) != len(want) { + t.Fatalf("expected %v, got %v", want, cpus) + } + for i := range want { + if cpus[i] != want[i] { + t.Fatalf("expected %v, got %v", want, cpus) + } + } +} + +// TestFlatTopology asserts the degraded fallback topology has the requested +// number of single-thread physical cores and logical CPUs, and that it is +// labelled degraded so placement code refuses it. +func TestFlatTopology(t *testing.T) { + topo := flatTopology(4) + if topo == nil { + t.Fatal("flatTopology returned nil") + } + if len(topo.Cores) != 4 { + t.Fatalf("expected 4 cores, got %d", len(topo.Cores)) + } + if topo.NumLCPUs != 4 { + t.Fatalf("expected 4 LCPUs, got %d", topo.NumLCPUs) + } + if !topo.Degraded { + t.Fatal("expected the synthesized topology to be marked degraded") + } + for _, pc := range topo.Cores { + if len(pc.Siblings) != 1 { + t.Fatalf("expected single-thread core, got siblings %v", pc.Siblings) + } + if !pc.L3Unknown { + t.Fatalf("synthesized core must not claim a known L3 id, got %+v", pc) + } + } +} + +// TestFlatTopology_MinimumOne guards against a non-positive core count. +func TestFlatTopology_MinimumOne(t *testing.T) { + topo := flatTopology(0) + if topo == nil || len(topo.Cores) != 1 { + t.Fatalf("expected 1 core fallback, got %+v", topo) + } +} + +// TestDiscoverTopology_DegradesLoudly confirms the documented contract: the +// topology is never nil, and every fallback also reports an error and sets +// Degraded, so a placement caller can refuse it instead of silently pinning +// against an invented model. +func TestDiscoverTopology_DegradesLoudly(t *testing.T) { + cases := []struct { + name string + rootFn func(t *testing.T) string + wantErr error // nil: any non-nil error accepted + }{ + { + name: "sysfs root does not exist", + rootFn: func(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "does-not-exist") + }, + }, + { + // The interesting case: nothing failed to read, sysfs simply + // described no CPUs. + name: "empty online file", + rootFn: func(t *testing.T) string { + t.Helper() + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "cpu", "online"), "\n") + return dir + }, + wantErr: ErrNoCPUTopology, + }, + { + name: "no cpuN directories", + rootFn: func(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "cpu"), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + return dir + }, + wantErr: ErrNoCPUTopology, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + saved := defaultSysfsRoot + defaultSysfsRoot = tc.rootFn(t) + defer func() { defaultSysfsRoot = saved }() + + topo, err := DiscoverTopology() + if topo == nil { + t.Fatalf("DiscoverTopology returned nil topology (err=%v)", err) + } + if err == nil { + t.Fatalf("expected an error reporting the degraded model, got nil") + } + if tc.wantErr != nil && !errors.Is(err, tc.wantErr) { + t.Fatalf("expected error %v, got %v", tc.wantErr, err) + } + if !topo.Degraded { + t.Fatal("expected Degraded to be set on the fallback topology") + } + if topo.NumLCPUs == 0 { + t.Fatalf("expected NumLCPUs>0, got 0") + } + }) + } +} diff --git a/pkg/pillar/cputopology/topology.go b/pkg/pillar/cputopology/topology.go new file mode 100644 index 00000000000..be1f785826a --- /dev/null +++ b/pkg/pillar/cputopology/topology.go @@ -0,0 +1,123 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package cputopology discovers native CPU topology (sockets, physical +// cores, SMT siblings, NUMA nodes, L3 cache domains) from Linux sysfs. +// +// It is intentionally dependency-light: pure Go, no CGO, stdlib only, and +// no dependency on any CPU allocator. It is meant to be imported by the +// CPU allocator, by the eve-k operator, and by the hardware-inventory +// (ZInfoMsg) path in zedagent. +package cputopology + +import "sort" + +// LCPU is a logical CPU id as the host kernel numbers it. +type LCPU uint32 + +// CoreInfo is one logical CPU's topology coordinates (sysfs reader output). +type CoreInfo struct { + LCore uint + Socket uint + CoreID uint + NUMA uint + L3ID uint + // L3Unknown is true when the platform does not expose an L3 cache id for + // this logical CPU (no level-3 cache index, or a cache index without an + // "id" file - common on ARM64). L3ID is then meaningless and must not be + // read as "L3 domain 0": that would make every core look like it shares + // one L3 domain. + L3Unknown bool +} + +// PhysicalCore is one physical core and its SMT sibling logical CPUs. +type PhysicalCore struct { + Socket uint + CoreID uint + NUMA uint + L3ID uint + L3Unknown bool // see CoreInfo.L3Unknown; true if unknown for any sibling + Siblings []LCPU // all logical CPUs on this physical core, sorted ascending +} + +// Topology is the discovered CPU topology, indexed for allocation. +type Topology struct { + Cores []PhysicalCore + ByLCPU map[LCPU]*PhysicalCore + NUMACores map[uint][]*PhysicalCore + L3Cores map[uint][]*PhysicalCore + NumLCPUs uint32 + // Degraded is true when the model was synthesized rather than read from + // sysfs. Such a model is fine for reporting but MUST NOT drive CPU + // placement: it invents CPU ids and claims every logical CPU is its own + // single-sibling core, so one-per-core placement would park nothing and + // hand real SMT siblings to two different workloads. + Degraded bool +} + +// coreKey groups logical CPUs into a physical core. SMT siblings are +// defined as logical CPUs sharing the same (Socket, CoreID) pair. This +// must NEVER be a cache/L2 id: Intel E-core modules share one L2 across +// four distinct physical cores, so grouping by L2 would wrongly merge +// unrelated cores into one. +type coreKey struct { + socket uint + coreID uint +} + +// BuildTopology groups logical CPU topology coordinates into physical +// cores and builds the lookup indices used by allocators. Repeated entries +// for the same logical CPU are ignored: a duplicate would otherwise show up +// as an extra SMT sibling and inflate the logical CPU count. +func BuildTopology(infos []CoreInfo) *Topology { + grouped := map[coreKey]*PhysicalCore{} + seen := map[uint]bool{} + for _, ci := range infos { + if seen[ci.LCore] { + continue + } + seen[ci.LCore] = true + k := coreKey{ci.Socket, ci.CoreID} + pc, ok := grouped[k] + if !ok { + pc = &PhysicalCore{Socket: ci.Socket, CoreID: ci.CoreID, NUMA: ci.NUMA, + L3ID: ci.L3ID, L3Unknown: ci.L3Unknown} + grouped[k] = pc + } + if ci.L3Unknown { + pc.L3Unknown = true + } + pc.Siblings = append(pc.Siblings, LCPU(ci.LCore)) + } + + topo := &Topology{ + ByLCPU: map[LCPU]*PhysicalCore{}, + NUMACores: map[uint][]*PhysicalCore{}, + L3Cores: map[uint][]*PhysicalCore{}, + } + keys := make([]coreKey, 0, len(grouped)) + for k := range grouped { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].socket != keys[j].socket { + return keys[i].socket < keys[j].socket + } + return keys[i].coreID < keys[j].coreID + }) + for _, k := range keys { + pc := grouped[k] + sort.Slice(pc.Siblings, func(i, j int) bool { return pc.Siblings[i] < pc.Siblings[j] }) + topo.Cores = append(topo.Cores, *pc) + } + for i := range topo.Cores { + pc := &topo.Cores[i] + for _, s := range pc.Siblings { + topo.ByLCPU[s] = pc + } + topo.NUMACores[pc.NUMA] = append(topo.NUMACores[pc.NUMA], pc) + topo.L3Cores[pc.L3ID] = append(topo.L3Cores[pc.L3ID], pc) + } + topo.NumLCPUs = uint32(len(topo.ByLCPU)) + return topo +} diff --git a/pkg/pillar/cputopology/topology_test.go b/pkg/pillar/cputopology/topology_test.go new file mode 100644 index 00000000000..8c62dd279c9 --- /dev/null +++ b/pkg/pillar/cputopology/topology_test.go @@ -0,0 +1,77 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cputopology + +import "testing" + +func TestBuildTopology_SiblingsByCoreID(t *testing.T) { + infos := []CoreInfo{ + {LCore: 0, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 4, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 1, Socket: 0, CoreID: 1, NUMA: 0, L3ID: 0}, + {LCore: 5, Socket: 0, CoreID: 1, NUMA: 0, L3ID: 0}, + } + + topo := BuildTopology(infos) + + if len(topo.Cores) != 2 { + t.Fatalf("expected 2 physical cores, got %d", len(topo.Cores)) + } + + pc0, ok := topo.ByLCPU[0] + if !ok { + t.Fatalf("expected ByLCPU to contain entry for LCPU 0") + } + if len(pc0.Siblings) != 2 || pc0.Siblings[0] != LCPU(0) || pc0.Siblings[1] != LCPU(4) { + t.Fatalf("expected siblings {0,4} for LCPU 0's core, got %v", pc0.Siblings) + } +} + +// TestBuildTopology_DuplicateCoreInfo asserts a repeated logical CPU neither +// inflates NumLCPUs nor appears twice as an SMT sibling: NumLCPUs is a count +// of distinct logical CPUs, not of input records. +func TestBuildTopology_DuplicateCoreInfo(t *testing.T) { + infos := []CoreInfo{ + {LCore: 0, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 4, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 0, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + } + + topo := BuildTopology(infos) + + if topo.NumLCPUs != 2 { + t.Fatalf("expected NumLCPUs == 2 for 2 distinct logical CPUs, got %d", topo.NumLCPUs) + } + if len(topo.Cores) != 1 { + t.Fatalf("expected 1 physical core, got %d", len(topo.Cores)) + } + if got := topo.Cores[0].Siblings; len(got) != 2 || got[0] != LCPU(0) || got[1] != LCPU(4) { + t.Fatalf("expected siblings {0,4}, got %v", got) + } +} + +func TestBuildTopology_SharedL2NotSiblings(t *testing.T) { + // Represents an E-core module: 4 distinct physical cores (CoreID 0..3) + // that happen to share the same socket/NUMA/L3 (as an E-core module + // would share an L2 cache). CoreInfo has no L2 field on purpose: SMT + // siblings must be grouped by (Socket, CoreID) only, never by any + // cache/L2 id. + infos := []CoreInfo{ + {LCore: 0, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 1, Socket: 0, CoreID: 1, NUMA: 0, L3ID: 0}, + {LCore: 2, Socket: 0, CoreID: 2, NUMA: 0, L3ID: 0}, + {LCore: 3, Socket: 0, CoreID: 3, NUMA: 0, L3ID: 0}, + } + + topo := BuildTopology(infos) + + if len(topo.Cores) != 4 { + t.Fatalf("expected 4 physical cores, got %d", len(topo.Cores)) + } + for _, pc := range topo.Cores { + if len(pc.Siblings) != 1 { + t.Fatalf("expected exactly 1 sibling per core, got %d for core %+v", len(pc.Siblings), pc) + } + } +} From 4456f6a4cdbbe452e871f39085a8716d87a7648c Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:00:15 +0000 Subject: [PATCH 02/15] cpuallocator: place workloads on whole cores, deterministically Replaces the CPU allocator with one that understands the machine's topology, so a workload asking for dedicated CPUs can be given whole physical cores in a NUMA-local, SMT-aware way rather than an arbitrary set of logical CPUs. Placement is computed for the whole set of pinned workloads at once and ordered by how constrained each one is -- whole-core-SMT first, since it can only use a core that really has two hardware threads and on a hybrid or SMT-disabled machine most cores cannot, then one-per-core, then anything thread-granular. The result is therefore a function of the request set rather than of the order requests arrive in. Allocating incrementally meant whichever workload activated first won the scarce cores, so a flexible workload could take the only SMT-capable core and leave a workload that needs one unplaceable -- and the same set of workloads could land differently on each boot. Plan does not mutate the allocator: the caller reserves an assignment when the workload actually starts, which is what lets a workload that has not started yet, or starts late, still claim the CPUs set aside for it. Score ranks an assignment by what actually costs performance -- NUMA nodes spanned, then last-level caches -- and deliberately not by which CPU indices were used. Many assignments share the best score, so comparing indices would report a workload as mis-placed merely because its first-choice CPUs were taken, and demand a restart that changes nothing. A core is withheld when any of its siblings is reserved for EVE. That costs capacity, so the shortage message says as much: handing out a core whose sibling runs housekeeping would reintroduce exactly the interference whole-core placement is bought to remove. The shortage also carries how many cores were needed against how many were free, so a caller can explain the refusal without computing a second, differently-filtered count. PoolUtilization reports the housekeeping, dedicated and isolated pools with both their CPU sets and their whole-core counts. Free threads alone answer "will it fit?" wrongly: threads left on partially-owned cores cannot satisfy a request for whole cores. Signed-off-by: Mikhail Malyshev --- pkg/pillar/cpuallocator/cpuallocator.go | 122 --- pkg/pillar/cpuallocator/cpuallocator_test.go | 206 ----- pkg/pillar/cpuallocator/placement.go | 665 +++++++++++++++ pkg/pillar/cpuallocator/placement_test.go | 814 +++++++++++++++++++ pkg/pillar/cpuallocator/plan.go | 183 +++++ pkg/pillar/cpuallocator/plan_test.go | 427 ++++++++++ pkg/pillar/cpuallocator/pools.go | 157 ++++ pkg/pillar/cpuallocator/pools_test.go | 185 +++++ 8 files changed, 2431 insertions(+), 328 deletions(-) delete mode 100644 pkg/pillar/cpuallocator/cpuallocator.go delete mode 100644 pkg/pillar/cpuallocator/cpuallocator_test.go create mode 100644 pkg/pillar/cpuallocator/placement.go create mode 100644 pkg/pillar/cpuallocator/placement_test.go create mode 100644 pkg/pillar/cpuallocator/plan.go create mode 100644 pkg/pillar/cpuallocator/plan_test.go create mode 100644 pkg/pillar/cpuallocator/pools.go create mode 100644 pkg/pillar/cpuallocator/pools_test.go diff --git a/pkg/pillar/cpuallocator/cpuallocator.go b/pkg/pillar/cpuallocator/cpuallocator.go deleted file mode 100644 index c1f46fc2881..00000000000 --- a/pkg/pillar/cpuallocator/cpuallocator.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (c) 2022 Zededa, Inc. -// SPDX-License-Identifier: Apache-2.0 - -package cpuallocator - -import ( - "fmt" - "sync" - - uuid "github.com/satori/go.uuid" -) - -type cpusList struct { - cpus []uint32 -} - -func (cpus *cpusList) contains(cpuToCheck uint32) bool { - for _, cpu := range cpus.cpus { - if cpu == cpuToCheck { - return true - } - } - return false -} - -// CPUAllocator stores information about the CPUs available in the system -// and provides interface to allocate and free them, per UUID. -type CPUAllocator struct { - sync.RWMutex // lock the access to the allocator - CPUsUsedByUUIDs map[uuid.UUID]cpusList // per UUID list of allocated CPUs - totalCPUs uint32 // total amount of CPUs in the system - numReservedForEVE uint32 // amount of the CPUs reserved for the EVE services -} - -// Init initializes a CPUAllocator instance. -// totalCPUs is the number of CPUs available is the system, -// numReserved is the number of CPUs considered to be always free, reserved for -// the EVE services and VMs with no CPU pinning enabled. -func Init(totalCPUs uint32, numReserved uint32) (*CPUAllocator, error) { - if totalCPUs == 0 || numReserved >= totalCPUs { - return nil, fmt.Errorf("invalid totalCPUs %d and/or numReserved %d", - totalCPUs, numReserved) - } - return &CPUAllocator{ - CPUsUsedByUUIDs: make(map[uuid.UUID]cpusList), - totalCPUs: totalCPUs, - numReservedForEVE: numReserved, - }, nil -} - -// Allocate a list of CPUs for a given uuid. If the amount of available CPUs is -// less than the requested amount (numCPUs), return an error and an empty list. -// If an allocation for a given uuid was already done before, also return an error -// and an empty list. -func (cpuAllocator *CPUAllocator) Allocate(uuid uuid.UUID, numCPUs int) ([]uint32, error) { - cpuAllocator.Lock() - defer cpuAllocator.Unlock() - if _, ok := cpuAllocator.CPUsUsedByUUIDs[uuid]; ok { - // Already allocated; return error - return []uint32{}, fmt.Errorf("multiple allocations for %s", uuid) - } - list, err := cpuAllocator.getFree(numCPUs) - if err != nil { - return list, err - } - cpuAllocator.CPUsUsedByUUIDs[uuid] = cpusList{cpus: list} - return list, nil -} - -// Free the CPUs previously allocated for a given uuid. -// Return an error for an attempt to free CPUs for a uuid that has no allocated CPUs. -func (cpuAllocator *CPUAllocator) Free(uuid uuid.UUID) error { - cpuAllocator.Lock() - defer cpuAllocator.Unlock() - if _, ok := cpuAllocator.CPUsUsedByUUIDs[uuid]; !ok { - // Nothing allocated; return error - return fmt.Errorf("free but no allocation for %s", uuid) - } - delete(cpuAllocator.CPUsUsedByUUIDs, uuid) - return nil -} - -func (cpuAllocator *CPUAllocator) usedByAnyUUID(cpuToCheck uint32) bool { - for _, cpus := range cpuAllocator.CPUsUsedByUUIDs { - if cpus.contains(cpuToCheck) { - return true - } - } - return false -} - -// Find the lowest numbered free CPUs, skipping the reserved ones -func (cpuAllocator *CPUAllocator) getFree(numCPUsRequested int) ([]uint32, error) { - result := make([]uint32, 0, cpuAllocator.totalCPUs) - found := 0 - var cpu uint32 - for cpu = cpuAllocator.numReservedForEVE; cpu < cpuAllocator.totalCPUs && found < numCPUsRequested; cpu++ { - if !cpuAllocator.usedByAnyUUID(cpu) { - result = append(result, cpu) - found++ - } - } - if found < numCPUsRequested { - return []uint32{}, fmt.Errorf("looking for %d CPUs only found %d reserved for EVE %d total %d", - numCPUsRequested, found, cpuAllocator.numReservedForEVE, cpuAllocator.totalCPUs) - } - return result, nil -} - -// GetAllFree returns all free CPUs (except the reserved ones) -func (cpuAllocator *CPUAllocator) GetAllFree() []uint32 { - cpuAllocator.RLock() - defer cpuAllocator.RUnlock() - result := make([]uint32, 0, cpuAllocator.totalCPUs) - var cpu uint32 - for cpu = 0; cpu < cpuAllocator.totalCPUs; cpu++ { - if !cpuAllocator.usedByAnyUUID(cpu) { - result = append(result, cpu) - } - } - return result -} diff --git a/pkg/pillar/cpuallocator/cpuallocator_test.go b/pkg/pillar/cpuallocator/cpuallocator_test.go deleted file mode 100644 index eb8d88f940a..00000000000 --- a/pkg/pillar/cpuallocator/cpuallocator_test.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright (c) 2022 Zededa, Inc. -// SPDX-License-Identifier: Apache-2.0 - -package cpuallocator - -import ( - "testing" - - uuid "github.com/satori/go.uuid" - "github.com/stretchr/testify/assert" -) - -func TestInit(t *testing.T) { - testMatrix := map[string]struct { - reservedCPUs uint32 - totalCPUs uint32 - expectInitFail bool - }{ - "init good": { - totalCPUs: 16, - reservedCPUs: 2, - }, - "init bad": { - totalCPUs: 16, - reservedCPUs: 32, - expectInitFail: true, - }, - } - for testname, test := range testMatrix { - t.Logf("Running test case %s", testname) - t.Run(testname, func(t *testing.T) { - ca, err := Init(test.totalCPUs, test.reservedCPUs) - if err != nil { - t.Logf("Init returned %s", err) - } - if test.expectInitFail { - assert.NotNil(t, err) - return - } - assert.Nil(t, err) - all := ca.GetAllFree() - t.Logf("GetAllFree returned %v", all) - assert.Equal(t, test.totalCPUs, uint32(len(all))) - }) - } -} - -type tm struct { - description string - uuid uuid.UUID - doFree bool // otherwise allocate - allocate int // number of CPUs - free int // number of CPUs - expectFail bool - expectAllocation []uint32 - expectAllFree []uint32 -} - -func TestAllocate(t *testing.T) { - uuid1, _ := uuid.NewV4() - uuid2, _ := uuid.NewV4() - uuid3, _ := uuid.NewV4() - uuid4, _ := uuid.NewV4() - uuid5, _ := uuid.NewV4() - uuid6, _ := uuid.NewV4() - uuid7, _ := uuid.NewV4() - - testSequence := make([]tm, 0) - testSequence = append(testSequence, - tm{ - description: "allocate bad", - uuid: uuid1, - allocate: 16, - expectFail: true, - expectAllFree: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, - }) - testSequence = append(testSequence, - tm{ - - description: "allocate good", - uuid: uuid1, - allocate: 8, - expectAllocation: []uint32{2, 3, 4, 5, 6, 7, 8, 9}, - expectAllFree: []uint32{0, 1, 10, 11, 12, 13, 14, 15}, - }) - testSequence = append(testSequence, - tm{ - description: "allocate too many", - uuid: uuid2, - allocate: 8, - expectFail: true, - expectAllFree: []uint32{0, 1, 10, 11, 12, 13, 14, 15}, - }) - testSequence = append(testSequence, - tm{ - description: "allocate less", - uuid: uuid2, - allocate: 2, - expectAllocation: []uint32{10, 11}, - expectAllFree: []uint32{0, 1, 12, 13, 14, 15}, - }) - testSequence = append(testSequence, - tm{ - description: "free 8", - uuid: uuid1, - doFree: true, - free: 8, // from "allocate good" above - expectAllFree: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15}, - }) - testSequence = append(testSequence, - tm{ - description: "allocate many after free", - uuid: uuid3, - allocate: 8, - expectAllocation: []uint32{2, 3, 4, 5, 6, 7, 8, 9}, - expectAllFree: []uint32{0, 1, 12, 13, 14, 15}, - }) - testSequence = append(testSequence, - tm{ - description: "allocate again", - uuid: uuid3, - allocate: 9, - expectFail: true, - expectAllFree: []uint32{0, 1, 12, 13, 14, 15}, - }) - testSequence = append(testSequence, - tm{ - description: "free without allocate", - uuid: uuid4, - doFree: true, - free: 0, - expectFail: true, - expectAllFree: []uint32{0, 1, 12, 13, 14, 15}, - }) - testSequence = append(testSequence, - tm{ - description: "double free 8", - uuid: uuid1, - doFree: true, - free: 8, // from "allocate good" above - expectFail: true, - expectAllFree: []uint32{0, 1, 12, 13, 14, 15}, - }) - testSequence = append(testSequence, - tm{ - description: "allocate remaining free", - uuid: uuid5, - allocate: 4, - expectAllocation: []uint32{12, 13, 14, 15}, - expectAllFree: []uint32{0, 1}, - }) - testSequence = append(testSequence, - tm{ - description: "allocate none", - uuid: uuid6, - allocate: 0, - expectAllocation: []uint32{}, - expectAllFree: []uint32{0, 1}, - }) - testSequence = append(testSequence, - tm{ - description: "allocate one", - uuid: uuid7, - allocate: 1, - expectFail: true, - expectAllFree: []uint32{0, 1}, - }) - - ca, err := Init(16, 2) - available := 16 // For GetAllFree - assert.Nil(t, err) - t.Logf("Running %d in sequence", len(testSequence)) - for _, test := range testSequence { - t.Logf("Running test case %s", test.description) - if test.doFree { - err := ca.Free(test.uuid) - if err != nil { - t.Logf("Free returned %s", err) - } - if test.expectFail { - assert.NotNil(t, err) - } else { - assert.Nil(t, err) - available += test.free - } - } else { - some, err := ca.Allocate(test.uuid, test.allocate) - if err != nil { - t.Logf("Allocate returned %s", err) - } - if test.expectFail { - assert.NotNil(t, err) - } else { - assert.Nil(t, err) - t.Logf("Allocate returned %v", some) - assert.Equal(t, test.allocate, len(some)) - assert.Equal(t, test.expectAllocation, some) - available -= test.allocate - } - } - all := ca.GetAllFree() - t.Logf("GetAllFree returned %v", all) - assert.Equal(t, available, len(all)) - assert.Equal(t, test.expectAllFree, all) - } -} diff --git a/pkg/pillar/cpuallocator/placement.go b/pkg/pillar/cpuallocator/placement.go new file mode 100644 index 00000000000..b38bd722e63 --- /dev/null +++ b/pkg/pillar/cpuallocator/placement.go @@ -0,0 +1,665 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cpuallocator + +import ( + "fmt" + "sort" + "strings" + "sync" + + "github.com/lf-edge/eve/pkg/pillar/cputopology" + uuid "github.com/satori/go.uuid" +) + +// PinMode selects how a pinned VM's vCPUs map onto physical cores. +type PinMode int + +const ( + ModeShared PinMode = iota // not topology-pinned (legacy shared pool) + ModeWholeCoreSMT // both SMT siblings of each core are vCPUs (guest threads=2) + ModeOnePerCore // one vCPU per physical core; sibling parked (guest threads=1) +) + +// String returns the policy name used in /persist pinning config and logs. +func (m PinMode) String() string { + switch m { + case ModeShared: + return "shared" + case ModeWholeCoreSMT: + return "whole-core-smt" + case ModeOnePerCore: + return "one-per-core" + default: + return fmt.Sprintf("PinMode(%d)", int(m)) + } +} + +// NUMAPolicy selects NUMA placement strictness. +type NUMAPolicy int + +const ( + NUMALocal NUMAPolicy = iota // all cores in one NUMA node, else NeedsRebalance (K8s single-numa-node/restricted) + NUMAAllowCross // no NUMA affinity; may span nodes freely (K8s none) + NUMABestEffort // prefer one NUMA node, fall back to spanning if it does not fit (K8s best-effort) +) + +// String returns the K8s Topology Manager policy name this maps to (also the +// value used in the /persist pinning config and logs). +func (n NUMAPolicy) String() string { + switch n { + case NUMALocal: + return "single-numa-node" + case NUMAAllowCross: + return "none" + case NUMABestEffort: + return "best-effort" + default: + return fmt.Sprintf("NUMAPolicy(%d)", int(n)) + } +} + +// Request is a single VM's placement request. +type Request struct { + UUID uuid.UUID + NumVCPUs int + Mode PinMode + NUMA NUMAPolicy +} + +// GuestTopology is the guest-visible -smp topology to emit. +type GuestTopology struct { + Sockets int + Cores int + Threads int +} + +// Assignment is the result of a successful placement. +type Assignment struct { + OrderedHostCPUs []cputopology.LCPU // guest vCPU i -> OrderedHostCPUs[i] + Guest GuestTopology + ParkedCPUs []cputopology.LCPU // siblings held idle (ModeOnePerCore) + NUMANodes []uint +} + +// Status is the outcome class of a placement attempt. +type Status int + +const ( + // StatusUnspecified is the zero value and means nothing was decided: no + // placement was attempted, or a Result was never filled in. Success must not + // be the zero value -- a Result that was dropped, defaulted or never + // produced would then read as a placement that worked, and a caller would + // pin a VM to an empty CPU set believing the allocator approved it. + StatusUnspecified Status = iota + Success + NeedsRebalance + Insufficient + InvalidRequest +) + +// Result carries the outcome of Allocate. +type Result struct { + Status Status + Assignment *Assignment + Message string + // TopologyUnsupported marks a request the node cannot satisfy in any + // arrangement; freeing CPUs will not help. It accompanies InvalidRequest. + // + // It exists to keep a caller from offering capacity advice that cannot + // apply: telling an operator to stop another workload or add cores is + // actively misleading when the machine simply has no core shaped the way the + // request needs (SMT disabled or absent, as on most ARM64 parts). + TopologyUnsupported bool + // CoresNeeded and CoresFree quantify a shortage: how many whole physical + // cores the request needs, and how many the allocator could have drawn on. + // Set only on the shortage statuses (Insufficient, NeedsRebalance), and only + // for the pinned modes -- a thread-granular request is not counted in cores. + // + // They exist so a caller can state the condition the shortage clears under + // without parsing Message. The published retry_condition has to give an + // operator the numbers ("needs 5 cores, 4 are free"), and reconstructing them + // from a second, independent computation would let the two disagree. + CoresNeeded int + CoresFree int +} + +// Placer owns dedicated-core bookkeeping and performs topology-aware +// placement. All methods are safe for concurrent use. +type Placer struct { + mu sync.Mutex + // topo and numReservedForEVE are write-once: set by the constructor and only + // read afterwards. That is the whole reason Plan and Score can run without + // holding mu. A method that refreshed or replaced the topology would break + // that and would have to take the lock here *and* in those two. + topo *cputopology.Topology + numReservedForEVE uint32 + // dedicated maps UUID -> every LCPU it holds (vCPU cores + parked). + // Guarded by mu. + dedicated map[uuid.UUID][]cputopology.LCPU +} + +// NewPlacer creates a Placer over the given topology, reserving the lowest +// numReservedForEVE logical CPUs for EVE housekeeping. +// +// Reserving as many CPUs as the host has (or more) leaves nothing to allocate, +// which is a misconfiguration of the eve_max_vcpus kernel argument rather than +// a runtime condition. It is rejected here so it surfaces once, at startup, +// instead of turning every later placement into an unexplained failure. +func NewPlacer(topo *cputopology.Topology, numReservedForEVE uint32) (*Placer, error) { + if topo == nil || topo.NumLCPUs == 0 { + return nil, fmt.Errorf("no CPU topology to place on") + } + if numReservedForEVE >= topo.NumLCPUs { + return nil, fmt.Errorf("%d CPUs reserved for EVE but the host has only %d: "+ + "no CPU would be left for workloads", numReservedForEVE, topo.NumLCPUs) + } + return newPlacer(topo, numReservedForEVE), nil +} + +// newPlacer builds a Placer without validation, for callers that derive one +// from an already-validated Placer. +func newPlacer(topo *cputopology.Topology, numReservedForEVE uint32) *Placer { + return &Placer{ + topo: topo, + numReservedForEVE: numReservedForEVE, + dedicated: map[uuid.UUID][]cputopology.LCPU{}, + } +} + +// Free releases all cores dedicated to id. Safe to call for an unknown id. +func (p *Placer) Free(id uuid.UUID) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.dedicated, id) +} + +// Reserve records that id already holds the given logical CPUs, without running +// placement. It is used to reseed a freshly created Placer from persisted +// DomainStatus after a domainmgr restart, so a VM that is already running does +// not have its dedicated cores handed to another VM. +// +// Unlike Allocate, the CPU set here comes from persisted state rather than from +// a placement decision, so it is validated. Reserve rejects an empty set, a +// logical CPU the host topology does not have, a CPU another workload already +// holds, and a second claim by the same id for a *different* set. Two statuses +// claiming one CPU would make HolderOf name an arbitrary owner and let the +// allocator hand the same CPU out twice, and the restart path is exactly where +// such a conflict is both plausible and least visible. +// +// Re-reserving the identical set (in any order) is a no-op returning nil, so +// replaying a status is harmless. On error nothing is recorded: a rejected +// Reserve leaves the placer exactly as it was. +func (p *Placer) Reserve(id uuid.UUID, cpus []uint32) error { + p.mu.Lock() + defer p.mu.Unlock() + if len(cpus) == 0 { + return fmt.Errorf("reserve %s: no CPUs given", id) + } + want := make([]cputopology.LCPU, 0, len(cpus)) + for _, c := range cpus { + lcpu := cputopology.LCPU(c) + if _, ok := p.topo.ByLCPU[lcpu]; !ok { + return fmt.Errorf("reserve %s: logical CPU %d is not in the host topology", id, c) + } + want = append(want, lcpu) + } + for _, c := range want { + if holder, taken := p.holderOf(c); taken && holder != id { + return fmt.Errorf("reserve %s: logical CPU %d is already held by %s", id, c, holder) + } + } + if held, ok := p.dedicated[id]; ok { + if !sameLCPUSet(held, want) { + return fmt.Errorf("reserve %s: already holds %v, refusing to re-reserve as %v", + id, held, want) + } + return nil + } + p.dedicated[id] = want + return nil +} + +// sameLCPUSet compares two CPU lists as multisets: a status replayed with its +// CPUs in a different order describes the same reservation. +func sameLCPUSet(a, b []cputopology.LCPU) bool { + if len(a) != len(b) { + return false + } + count := make(map[cputopology.LCPU]int, len(a)) + for _, c := range a { + count[c]++ + } + for _, c := range b { + count[c]-- + if count[c] < 0 { + return false + } + } + return true +} + +// DedicatedSet returns the union of all dedicated LCPUs (vCPU + parked), +// sorted ascending. +func (p *Placer) DedicatedSet() []cputopology.LCPU { + p.mu.Lock() + defer p.mu.Unlock() + seen := map[cputopology.LCPU]bool{} + for _, cs := range p.dedicated { + for _, c := range cs { + seen[c] = true + } + } + out := make([]cputopology.LCPU, 0, len(seen)) + for c := range seen { + out = append(out, c) + } + sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) + return out +} + +// HolderOf reports which workload currently holds a logical CPU, if any. +// +// It exists so a refused placement can name the workloads standing on the CPUs +// the plan set aside for it. Without a name the operator is told a repack would +// help but not what to restart, which is the difference between an actionable +// report and a shrug. +func (p *Placer) HolderOf(cpu cputopology.LCPU) (uuid.UUID, bool) { + p.mu.Lock() + defer p.mu.Unlock() + return p.holderOf(cpu) +} + +// holderOf is HolderOf without locking. Caller must hold p.mu. +func (p *Placer) holderOf(cpu cputopology.LCPU) (uuid.UUID, bool) { + for id, cs := range p.dedicated { + for _, c := range cs { + if c == cpu { + return id, true + } + } + } + return uuid.UUID{}, false +} + +// coreIsPartlyReserved reports whether any sibling of a physical core falls in +// the EVE-reserved low range. +// +// Such a core is withheld whole, not per sibling. Topology placement hands out +// physical cores, and a core whose other thread runs EVE housekeeping is not a +// core the workload owns exclusively -- handing it out would give back exactly +// the SMT interference full-pcpus-only exists to remove. The cost is that +// reserving N logical CPUs can withhold up to N whole cores (2N logical CPUs on +// an SMT host), because the low CPU numbers Linux assigns usually land on +// distinct cores. Operators sizing eve_max_vcpus must account for that; the +// Insufficient message spells it out when it bites. +func (p *Placer) coreIsPartlyReserved(pc *cputopology.PhysicalCore) bool { + for _, s := range pc.Siblings { + if uint32(s) < p.numReservedForEVE { + return true + } + } + return false +} + +// coreIsDedicated reports whether any sibling of a physical core is already +// held by some workload. +func coreIsDedicated(pc *cputopology.PhysicalCore, dedicated map[cputopology.LCPU]bool) bool { + for _, s := range pc.Siblings { + if dedicated[s] { + return true + } + } + return false +} + +// dedicatedLookup returns a membership set of all dedicated LCPUs. +func (p *Placer) dedicatedLookup() map[cputopology.LCPU]bool { + m := map[cputopology.LCPU]bool{} + for _, cs := range p.dedicated { + for _, c := range cs { + m[c] = true + } + } + return m +} + +// AllocateShared allocates n lowest-numbered free logical CPUs for a legacy +// (non-topology) pinned VM. Recorded in the same bookkeeping as topology +// allocations, so the two can never overlap. Skips the EVE-reserved low range. +func (p *Placer) AllocateShared(id uuid.UUID, n int) ([]cputopology.LCPU, error) { + p.mu.Lock() + defer p.mu.Unlock() + if _, ok := p.dedicated[id]; ok { + return nil, fmt.Errorf("multiple allocations for %s", id) + } + if n <= 0 { + return nil, fmt.Errorf("AllocateShared: n must be > 0") + } + ded := p.dedicatedLookup() + var picked []cputopology.LCPU + for _, c := range p.allLCPUsSorted() { + if uint32(c) < p.numReservedForEVE || ded[c] { + continue + } + picked = append(picked, c) + if len(picked) == n { + break + } + } + if len(picked) < n { + return nil, fmt.Errorf("insufficient CPUs: need %d, have %d free", n, len(picked)) + } + p.dedicated[id] = picked + return picked, nil +} + +// FreeCPUs returns all logical CPUs not dedicated to any VM (topology OR +// shared), INCLUDING the EVE-reserved low range — matching the legacy +// GetAllFree semantics used for non-pinned VM cpusets and emulator housekeeping. +func (p *Placer) FreeCPUs() []cputopology.LCPU { + p.mu.Lock() + defer p.mu.Unlock() + ded := p.dedicatedLookup() + var out []cputopology.LCPU + for _, c := range p.allLCPUsSorted() { + if !ded[c] { + out = append(out, c) + } + } + return out +} + +// allLCPUsSorted returns every logical CPU in the topology, ascending. +// Caller must hold p.mu. +func (p *Placer) allLCPUsSorted() []cputopology.LCPU { + var all []cputopology.LCPU + for i := range p.topo.Cores { + all = append(all, p.topo.Cores[i].Siblings...) + } + sort.Slice(all, func(i, j int) bool { return all[i] < all[j] }) + return all +} + +// Allocate performs topology-aware placement for one VM. On Success it commits +// the allocation into the placer's bookkeeping -- the returned CPUs (vCPU and +// parked alike) are held by r.UUID until Free -- so a caller that then discards +// the Result must Free it. Every other status leaves the placer untouched. +func (p *Placer) Allocate(r Request) Result { + p.mu.Lock() + defer p.mu.Unlock() + + if _, ok := p.dedicated[r.UUID]; ok { + return Result{Status: InvalidRequest, Message: fmt.Sprintf("already allocated for %s", r.UUID)} + } + if r.NumVCPUs <= 0 { + return Result{Status: InvalidRequest, Message: "NumVCPUs must be > 0"} + } + + var coresNeeded, threads int + switch r.Mode { + case ModeWholeCoreSMT: + if r.NumVCPUs%2 != 0 { + return Result{Status: InvalidRequest, Message: "whole-core-smt requires an even vCPU count"} + } + coresNeeded = r.NumVCPUs / 2 + threads = 2 + case ModeOnePerCore: + coresNeeded = r.NumVCPUs + threads = 1 + default: + return Result{Status: InvalidRequest, Message: "Allocate is only for pinned modes"} + } + + // A whole-core-SMT request on a node where no core has the required thread + // count is unsatisfiable, not short of capacity: the loop below would skip + // every core and report a shortage of cores that could never help. Separating + // the two matters because the shortage advice -- stop a workload, add cores -- + // is false on an SMT-disabled or non-SMT host, which is the common case on + // ARM64 and the most likely way the default policy fails. + if r.Mode == ModeWholeCoreSMT && !p.anyCoreHasThreads(threads) { + return Result{ + Status: InvalidRequest, + TopologyUnsupported: true, + Message: topologyUnsupportedMessage(p.topo, threads), + } + } + + dedicated := p.dedicatedLookup() + + // Free cores grouped by NUMA node, preserving deterministic order. Cores + // skipped for a structural reason are counted so a shortage can say why: + // a bare "insufficient" on a host with visibly idle CPUs is unactionable. + freeByNUMA := map[uint][]*cputopology.PhysicalCore{} + numaOrder := []uint{} + totalFree, partlyReserved, notSMT := 0, 0, 0 + for i := range p.topo.Cores { + pc := &p.topo.Cores[i] + if p.coreIsPartlyReserved(pc) { + partlyReserved++ + continue + } + if coreIsDedicated(pc, dedicated) { + continue + } + // whole-core-smt maps one vCPU onto each SMT sibling, so it can only use + // a core with exactly that many threads. On hybrid parts a single-threaded + // core (e.g. an E-core) cannot present threads=2; skip it here rather than + // emit an assignment with fewer host CPUs than vCPUs. + if r.Mode == ModeWholeCoreSMT && len(pc.Siblings) != threads { + notSMT++ + continue + } + if _, ok := freeByNUMA[pc.NUMA]; !ok { + numaOrder = append(numaOrder, pc.NUMA) + } + freeByNUMA[pc.NUMA] = append(freeByNUMA[pc.NUMA], pc) + totalFree++ + } + sort.Slice(numaOrder, func(i, j int) bool { return numaOrder[i] < numaOrder[j] }) + + if totalFree < coresNeeded { + return Result{ + Status: Insufficient, + Message: shortageMessage(coresNeeded, totalFree, partlyReserved, notSMT, threads), + CoresNeeded: coresNeeded, + CoresFree: totalFree, + } + } + + // allNodes may assume sufficiency: total free is already >= coresNeeded. + singleNode := func() []*cputopology.PhysicalCore { + for _, n := range numaOrder { + if cand := freeByNUMA[n]; len(cand) >= coresNeeded { + return pickCores(cand, coresNeeded) + } + } + return nil + } + allNodes := func() []*cputopology.PhysicalCore { + var all []*cputopology.PhysicalCore + for _, n := range numaOrder { + all = append(all, freeByNUMA[n]...) + } + return pickCores(all, coresNeeded) + } + + var chosen []*cputopology.PhysicalCore + switch r.NUMA { + case NUMALocal: + if chosen = singleNode(); chosen == nil { + return Result{ + Status: NeedsRebalance, + Message: fmt.Sprintf("need %d cores in one NUMA node; none has enough (total free %d)", + coresNeeded, totalFree), + CoresNeeded: coresNeeded, + CoresFree: totalFree, + } + } + case NUMABestEffort: + if chosen = singleNode(); chosen == nil { + chosen = allNodes() + } + default: // NUMAAllowCross + chosen = allNodes() + } + + var ordered, parked []cputopology.LCPU + nodeSet := map[uint]bool{} + for _, pc := range chosen { + nodeSet[pc.NUMA] = true + switch r.Mode { + case ModeWholeCoreSMT: + ordered = append(ordered, pc.Siblings...) + case ModeOnePerCore: + ordered = append(ordered, pc.Siblings[0]) + if len(pc.Siblings) > 1 { + parked = append(parked, pc.Siblings[1:]...) + } + } + } + + nodes := make([]uint, 0, len(nodeSet)) + for n := range nodeSet { + nodes = append(nodes, n) + } + sort.Slice(nodes, func(i, j int) bool { return nodes[i] < nodes[j] }) + + assignment := &Assignment{ + OrderedHostCPUs: ordered, + Guest: GuestTopology{Sockets: 1, Cores: coresNeeded, Threads: threads}, + ParkedCPUs: parked, + NUMANodes: nodes, + } + if err := assignment.validate(p.topo); err != nil { + return Result{ + Status: InvalidRequest, + Message: fmt.Sprintf("internal error: computed placement is inconsistent: %v", err), + } + } + + p.dedicated[r.UUID] = append(append([]cputopology.LCPU{}, ordered...), parked...) + return Result{Status: Success, Assignment: assignment} +} + +// validate checks a computed assignment for internal consistency before it is +// handed out. The guest -smp topology and the host CPU list are two views of one +// decision, and nothing downstream cross-checks them: today a mismatch surfaces +// only when QEMU refuses the vCPU count, i.e. as a failed domain start with no +// indication of which side was wrong. +// +// The topology is a parameter because NUMANodes cannot be verified from the +// assignment alone -- it is a property of the host CPUs that were picked. +func (a *Assignment) validate(topo *cputopology.Topology) error { + if want := a.Guest.Sockets * a.Guest.Cores * a.Guest.Threads; want != len(a.OrderedHostCPUs) { + return fmt.Errorf("guest topology %d/%d/%d needs %d vCPUs but %d host CPUs were assigned", + a.Guest.Sockets, a.Guest.Cores, a.Guest.Threads, want, len(a.OrderedHostCPUs)) + } + vcpus := make(map[cputopology.LCPU]bool, len(a.OrderedHostCPUs)) + nodes := map[uint]bool{} + for _, c := range a.OrderedHostCPUs { + if vcpus[c] { + return fmt.Errorf("host CPU %d is assigned to more than one vCPU", c) + } + vcpus[c] = true + pc, ok := topo.ByLCPU[c] + if !ok { + return fmt.Errorf("host CPU %d is not in the host topology", c) + } + nodes[pc.NUMA] = true + } + for _, c := range a.ParkedCPUs { + if vcpus[c] { + return fmt.Errorf("host CPU %d is both a vCPU and parked", c) + } + } + if len(a.NUMANodes) != len(nodes) { + return fmt.Errorf("NUMANodes %v does not match the nodes of the assigned CPUs", a.NUMANodes) + } + for _, n := range a.NUMANodes { + if !nodes[n] { + return fmt.Errorf("NUMANodes %v does not match the nodes of the assigned CPUs", a.NUMANodes) + } + } + return nil +} + +// anyCoreHasThreads reports whether the topology has at least one physical core +// presenting exactly n hardware threads, regardless of whether it is free. +func (p *Placer) anyCoreHasThreads(n int) bool { + for i := range p.topo.Cores { + if len(p.topo.Cores[i].Siblings) == n { + return true + } + } + return false +} + +// topologyUnsupportedMessage explains why no arrangement of this node's CPUs can +// present the requested thread count, naming the thread counts it does have so +// the operator can tell an SMT-less machine from a misconfigured one. +func topologyUnsupportedMessage(topo *cputopology.Topology, threads int) string { + counts := map[int]bool{} + for i := range topo.Cores { + counts[len(topo.Cores[i].Siblings)] = true + } + have := make([]int, 0, len(counts)) + for n := range counts { + have = append(have, n) + } + sort.Ints(have) + + detail := fmt.Sprintf("every core has %s", threadCountPhrase(have)) + if len(have) == 1 && have[0] == 1 { + detail = "no core has an SMT sibling (SMT is disabled or the CPU has none)" + } + return fmt.Sprintf("whole-core-smt needs a physical core with %d hardware threads, but %s: "+ + "threads=%d cannot be presented in any arrangement, so freeing CPUs will not help", + threads, detail, threads) +} + +// threadCountPhrase renders the thread counts a node's cores have, e.g. +// "1 thread" or "1 or 4 threads". +func threadCountPhrase(counts []int) string { + if len(counts) == 1 && counts[0] == 1 { + return "1 thread" + } + parts := make([]string, 0, len(counts)) + for _, n := range counts { + parts = append(parts, fmt.Sprint(n)) + } + return strings.Join(parts, " or ") + " threads" +} + +// shortageMessage explains a core shortage, naming the structural reasons that +// removed cores from the pool. Without them the operator sees idle CPUs in top +// and an "insufficient" from EVE, and has no way to connect the two. +func shortageMessage(needed, free, partlyReserved, notSMT, threads int) string { + msg := fmt.Sprintf("need %d free cores, have %d", needed, free) + var because []string + if partlyReserved > 0 { + because = append(because, fmt.Sprintf( + "%d cores are partly reserved for EVE and cannot be handed out whole", + partlyReserved)) + } + if notSMT > 0 { + // Not "have no SMT sibling": a core is skipped whenever its thread count + // differs from the request, which includes 4-thread cores that do have + // siblings. + because = append(because, fmt.Sprintf( + "%d cores do not have exactly %d hardware threads and cannot present threads=%d", + notSMT, threads, threads)) + } + if len(because) > 0 { + msg += " (" + strings.Join(because, "; ") + ")" + } + return msg +} + +// pickCores returns the first n cores from an already-deterministic slice. +func pickCores(cores []*cputopology.PhysicalCore, n int) []*cputopology.PhysicalCore { + out := make([]*cputopology.PhysicalCore, n) + copy(out, cores[:n]) + return out +} diff --git a/pkg/pillar/cpuallocator/placement_test.go b/pkg/pillar/cpuallocator/placement_test.go new file mode 100644 index 00000000000..731461d80fa --- /dev/null +++ b/pkg/pillar/cpuallocator/placement_test.go @@ -0,0 +1,814 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cpuallocator + +import ( + "strings" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/cputopology" + uuid "github.com/satori/go.uuid" +) + +func u(s string) uuid.UUID { return uuid.NewV5(uuid.NamespaceOID, s) } + +func mustPlacer(t *testing.T, topo *cputopology.Topology, reserved uint32) *Placer { + t.Helper() + p, err := NewPlacer(topo, reserved) + if err != nil { + t.Fatalf("NewPlacer(reserved=%d): %v", reserved, err) + } + return p +} + +// two physical cores, SMT2, single socket/NUMA/L3 +func twoCoresSMT2() *cputopology.Topology { + return cputopology.BuildTopology([]cputopology.CoreInfo{ + {LCore: 0, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 4, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 1, Socket: 0, CoreID: 1, NUMA: 0, L3ID: 0}, + {LCore: 5, Socket: 0, CoreID: 1, NUMA: 0, L3ID: 0}, + }) +} + +func TestPlacer_FreeAndDedicatedSet(t *testing.T) { + p := mustPlacer(t, twoCoresSMT2(), 0) + if len(p.DedicatedSet()) != 0 { + t.Fatalf("fresh placer must have empty dedicated set, got %v", p.DedicatedSet()) + } + p.Free(u("nobody")) // must not panic on unknown uuid + if len(p.DedicatedSet()) != 0 { + t.Fatalf("still empty after Free of unknown uuid") + } +} + +// 2 sockets x 4 physical cores x SMT2, distinct L3/NUMA per socket. +func twoSocketTopo() *cputopology.Topology { + var infos []cputopology.CoreInfo + lc := uint(0) + for socket := uint(0); socket < 2; socket++ { + for core := uint(0); core < 4; core++ { + for thread := 0; thread < 2; thread++ { + infos = append(infos, cputopology.CoreInfo{ + LCore: lc, Socket: socket, CoreID: core, NUMA: socket, L3ID: socket, + }) + lc++ + } + } + } + return cputopology.BuildTopology(infos) +} + +// one physical core per NUMA node (forces NeedsRebalance for a 2-core request). +func twoNodesOneCoreEach() *cputopology.Topology { + return cputopology.BuildTopology([]cputopology.CoreInfo{ + {LCore: 0, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 1, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 2, Socket: 1, CoreID: 0, NUMA: 1, L3ID: 1}, + {LCore: 3, Socket: 1, CoreID: 0, NUMA: 1, L3ID: 1}, + }) +} + +func TestAllocate_WholeCoreSMT_NUMALocal(t *testing.T) { + topo := twoSocketTopo() + p := mustPlacer(t, topo, 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r.Status != Success { + t.Fatalf("want Success, got %v (%s)", r.Status, r.Message) + } + a := r.Assignment + if a.Guest != (GuestTopology{Sockets: 1, Cores: 2, Threads: 2}) { + t.Fatalf("guest topo want 1/2/2, got %+v", a.Guest) + } + if len(a.OrderedHostCPUs) != 4 { + t.Fatalf("want 4 host cpus, got %d", len(a.OrderedHostCPUs)) + } + n := topo.ByLCPU[a.OrderedHostCPUs[0]].NUMA + for _, c := range a.OrderedHostCPUs { + if topo.ByLCPU[c].NUMA != n { + t.Fatalf("NUMA-local violated: %v", a.OrderedHostCPUs) + } + } + // vCPU pair (0,1) must be SMT siblings (same physical core). + if topo.ByLCPU[a.OrderedHostCPUs[0]].CoreID != topo.ByLCPU[a.OrderedHostCPUs[1]].CoreID { + t.Fatalf("vcpu pair 0,1 not sibling-mapped: %v", a.OrderedHostCPUs) + } +} + +func TestAllocate_OddRejectedForSMT(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 3, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r.Status != InvalidRequest { + t.Fatalf("odd vcpu in SMT mode must be InvalidRequest, got %v", r.Status) + } +} + +func TestAllocate_OnePerCore_ParksSiblings(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 2, Mode: ModeOnePerCore, NUMA: NUMALocal}) + if r.Status != Success { + t.Fatalf("want Success, got %v (%s)", r.Status, r.Message) + } + if r.Assignment.Guest.Threads != 1 { + t.Fatalf("one-per-core guest threads must be 1, got %d", r.Assignment.Guest.Threads) + } + if len(r.Assignment.ParkedCPUs) != 2 { + t.Fatalf("want 2 parked siblings, got %v", r.Assignment.ParkedCPUs) + } + if len(p.DedicatedSet()) != 4 { + t.Fatalf("dedicated set must include parked siblings (want 4), got %d", len(p.DedicatedSet())) + } +} + +func TestAllocate_NoCrossVMCoreSharing(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) + if r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}); r.Status != Success { + t.Fatalf("vm1 should succeed, got %v", r.Status) + } + r2 := p.Allocate(Request{UUID: u("vm2"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r2.Status != Success { + t.Fatalf("vm2 should succeed (room remains), got %v (%s)", r2.Status, r2.Message) + } + seen := map[cputopology.LCPU]bool{} + for _, c := range p.dedicated[u("vm1")] { + seen[c] = true + } + for _, c := range p.dedicated[u("vm2")] { + if seen[c] { + t.Fatalf("cross-VM core sharing at lcpu %d", c) + } + } +} + +func TestAllocate_NeedsRebalance(t *testing.T) { + // Two NUMA nodes, one core each; a 2-core NUMA-local request can't fit in + // one node even though total free (2) is enough. + p := mustPlacer(t, twoNodesOneCoreEach(), 0) + r := p.Allocate(Request{UUID: u("b"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r.Status != NeedsRebalance { + t.Fatalf("want NeedsRebalance, got %v (%s)", r.Status, r.Message) + } + // The published retry condition quotes these numbers, so they are part of the + // contract, not a debugging aid: a rebalance needs 2 cores and 2 are free, + // just not in one node. + if r.CoresNeeded != 2 || r.CoresFree != 2 { + t.Errorf("want CoresNeeded=2 CoresFree=2, got %d/%d", r.CoresNeeded, r.CoresFree) + } +} + +func TestAllocate_InsufficientTotal(t *testing.T) { + p := mustPlacer(t, twoNodesOneCoreEach(), 0) + // 6 vCPUs = 3 cores, but only 2 physical cores exist anywhere. + r := p.Allocate(Request{UUID: u("x"), NumVCPUs: 6, Mode: ModeWholeCoreSMT, NUMA: NUMAAllowCross}) + if r.Status != Insufficient { + t.Fatalf("want Insufficient, got %v (%s)", r.Status, r.Message) + } + if r.CoresNeeded != 3 || r.CoresFree != 2 { + t.Errorf("want CoresNeeded=3 CoresFree=2, got %d/%d", r.CoresNeeded, r.CoresFree) + } +} + +func oneCoreTopo() *cputopology.Topology { + return cputopology.BuildTopology([]cputopology.CoreInfo{ + {LCore: 0, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 1, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + }) +} + +func TestAllocate_ReservedCPUsExcluded(t *testing.T) { + // twoCoresSMT2 cores: c0={0,4}, c1={1,5}. Reserve lcpu 0 -> core {0,4} + // excluded wholesale, leaving 1 free core; a 2-core request must not fit. + p := mustPlacer(t, twoCoresSMT2(), 1) + if r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}); r.Status != Insufficient { + t.Fatalf("reserved core must be excluded -> Insufficient, got %v (%s)", r.Status, r.Message) + } + r := p.Allocate(Request{UUID: u("vm2"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r.Status != Success { + t.Fatalf("1-core request should fit on the non-reserved core, got %v (%s)", r.Status, r.Message) + } + for _, c := range r.Assignment.OrderedHostCPUs { + if c == 0 || c == 4 { + t.Fatalf("allocated a reserved core lcpu %d", c) + } + } +} + +// fourCoresSMT2Interleaved is an 8-CPU / 4-core SMT2 host numbered the way +// Linux normally numbers one: lcpu 0..3 are the first thread of cores 0..3 and +// lcpu 4..7 the second. Reserving the lowest N CPUs therefore touches N +// *different* cores, which is what makes the whole-core reservation rule +// visible. +func fourCoresSMT2Interleaved() *cputopology.Topology { + var infos []cputopology.CoreInfo + for thread := uint(0); thread < 2; thread++ { + for core := uint(0); core < 4; core++ { + infos = append(infos, cputopology.CoreInfo{ + LCore: thread*4 + core, Socket: 0, CoreID: core, NUMA: 0, L3ID: 0, + }) + } + } + return cputopology.BuildTopology(infos) +} + +// Reserving CPUs for EVE withholds every physical core they sit on, whole. On +// an SMT host with the usual Linux numbering, cpusReserved=2 therefore costs +// two cores (four logical CPUs), not two logical CPUs -- so a 6-vCPU whole-core +// request does not fit on 8 CPUs. That is deliberate (a core EVE shares is not +// a core a workload owns exclusively) and the shortage must say so. +func TestAllocate_ReservedRangeWithholdsWholeCores(t *testing.T) { + p := mustPlacer(t, fourCoresSMT2Interleaved(), 2) // lcpu 0 and 1 -> cores 0 and 1 + r := p.Allocate(Request{UUID: u("big"), NumVCPUs: 6, Mode: ModeWholeCoreSMT, NUMA: NUMAAllowCross}) + if r.Status != Insufficient { + t.Fatalf("6 vCPUs must not fit behind 2 reserved CPUs, got %v (%s)", r.Status, r.Message) + } + if !strings.Contains(r.Message, "partly reserved") { + t.Errorf("shortage must explain the reserved cores, got %q", r.Message) + } + // Exactly the two untouched cores remain usable. + fits := p.Allocate(Request{UUID: u("fits"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMAAllowCross}) + if fits.Status != Success { + t.Fatalf("the 2 fully free cores must still be allocatable, got %v (%s)", + fits.Status, fits.Message) + } + for _, c := range fits.Assignment.OrderedHostCPUs { + if pc := p.topo.ByLCPU[c]; pc.CoreID < 2 { + t.Errorf("allocated core %d, which shares a thread with the reserved range", pc.CoreID) + } + } +} + +// A shortage caused by hybrid single-thread cores must name that reason too, and +// must say it in terms of the thread count that was skipped: cores are dropped +// whenever their thread count differs from the request, which also covers +// 4-thread cores that do have siblings. +func TestAllocate_ShortageNamesCoresWithWrongThreadCount(t *testing.T) { + p := mustPlacer(t, hybridTopo(), 0) + r := p.Allocate(Request{UUID: u("smt"), NumVCPUs: 8, Mode: ModeWholeCoreSMT, NUMA: NUMAAllowCross}) + if r.Status != Insufficient { + t.Fatalf("want Insufficient, got %v (%s)", r.Status, r.Message) + } + if r.TopologyUnsupported { + t.Error("SMT-capable cores exist, so this is a shortage, not an unsupported request") + } + if !strings.Contains(r.Message, "do not have exactly 2 hardware threads") { + t.Errorf("shortage must explain the skipped cores, got %q", r.Message) + } + // 8 vCPUs = 4 cores; only the two SMT P-cores can serve them. + if r.CoresNeeded != 4 || r.CoresFree != 2 { + t.Errorf("want CoresNeeded=4 CoresFree=2, got %d/%d", r.CoresNeeded, r.CoresFree) + } +} + +// Reserving every CPU for EVE is a misconfigured eve_max_vcpus, not a runtime +// state: it must fail at construction rather than turn every later placement +// into an unexplained shortage. +func TestNewPlacer_RejectsOverReservation(t *testing.T) { + topo := fourCoresSMT2Interleaved() // 8 logical CPUs + for _, reserved := range []uint32{8, 9, 100} { + if _, err := NewPlacer(topo, reserved); err == nil { + t.Errorf("reserving %d of 8 CPUs must be rejected", reserved) + } + } + if _, err := NewPlacer(topo, 7); err != nil { + t.Errorf("reserving 7 of 8 CPUs still leaves one, must be accepted: %v", err) + } + if _, err := NewPlacer(nil, 0); err == nil { + t.Error("a nil topology must be rejected") + } +} + +func TestAllocate_DoubleAllocateRejected(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) + if r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}); r.Status != Success { + t.Fatalf("first allocate should succeed, got %v", r.Status) + } + if r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}); r.Status != InvalidRequest { + t.Fatalf("re-allocating same UUID must be InvalidRequest, got %v (%s)", r.Status, r.Message) + } +} + +func TestAllocate_AllowCrossSpansNodes(t *testing.T) { + p := mustPlacer(t, twoNodesOneCoreEach(), 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMAAllowCross}) + if r.Status != Success { + t.Fatalf("allow-cross should span both nodes and succeed, got %v (%s)", r.Status, r.Message) + } + if len(r.Assignment.NUMANodes) != 2 { + t.Fatalf("want assignment spanning 2 NUMA nodes, got %v", r.Assignment.NUMANodes) + } + if len(r.Assignment.OrderedHostCPUs) != 4 { + t.Fatalf("want 4 host cpus, got %d", len(r.Assignment.OrderedHostCPUs)) + } +} + +func TestAllocate_ParkedSiblingBlocksReuse(t *testing.T) { + p := mustPlacer(t, oneCoreTopo(), 0) + if r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 1, Mode: ModeOnePerCore, NUMA: NUMALocal}); r.Status != Success { + t.Fatalf("first one-per-core should succeed, got %v", r.Status) + } + r := p.Allocate(Request{UUID: u("vm2"), NumVCPUs: 1, Mode: ModeOnePerCore, NUMA: NUMALocal}) + if r.Status != Insufficient { + t.Fatalf("parked sibling must block core reuse -> Insufficient, got %v (%s)", r.Status, r.Message) + } +} + +func TestAllocateShared_Basic(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) + got, err := p.AllocateShared(u("legacy"), 3) + if err != nil || len(got) != 3 { + t.Fatalf("want 3 cpus, got %v err %v", got, err) + } + if len(p.DedicatedSet()) != 3 { + t.Fatalf("shared alloc must be in dedicated set") + } +} + +func TestAllocateShared_SkipsReserved(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 2) // reserve lcpus 0,1 + got, _ := p.AllocateShared(u("legacy"), 1) + if got[0] < 2 { + t.Fatalf("must skip reserved cpus 0,1, got %v", got) + } +} + +func TestMixed_NoOverlap_TopologyThenShared(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r.Status != Success { + t.Fatal(r.Message) + } + // collect vm1's cpus + vm1 := map[uint32]bool{} + for _, c := range p.dedicated[u("vm1")] { + vm1[uint32(c)] = true + } + got, err := p.AllocateShared(u("legacy"), 8) + if err != nil { + t.Fatalf("legacy alloc should fit remaining cores: %v", err) + } + for _, c := range got { + if vm1[uint32(c)] { + t.Fatalf("shared alloc reused topology-dedicated cpu %d", c) + } + } +} + +func TestMixed_NoOverlap_SharedThenTopology(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) + shared, _ := p.AllocateShared(u("legacy"), 2) + sharedSet := map[uint32]bool{} + for _, c := range shared { + sharedSet[uint32(c)] = true + } + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r.Status != Success { + t.Fatal(r.Message) + } + for _, c := range r.Assignment.OrderedHostCPUs { + if sharedSet[uint32(c)] { + t.Fatalf("topology alloc reused shared cpu %d", c) + } + } +} + +// hybridTopo mirrors an Intel hybrid part: two SMT2 P-cores (lcpu 0/1 on core +// 0, 2/3 on core 1) plus four single-thread E-cores (lcpu 4..7 on cores 2..5), +// single socket/NUMA/L3. +func hybridTopo() *cputopology.Topology { + infos := []cputopology.CoreInfo{ + {LCore: 0, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 1, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 2, Socket: 0, CoreID: 1, NUMA: 0, L3ID: 0}, + {LCore: 3, Socket: 0, CoreID: 1, NUMA: 0, L3ID: 0}, + } + for i := 0; i < 4; i++ { + infos = append(infos, cputopology.CoreInfo{ + LCore: uint(4 + i), Socket: 0, CoreID: uint(2 + i), NUMA: 0, L3ID: 0, + }) + } + return cputopology.BuildTopology(infos) +} + +// whole-core-smt must place a vCPU on each SMT sibling of ONE physical core and +// never satisfy the request with a single-thread (E) core, which cannot present +// threads=2. Regression for the hybrid-CPU count mismatch (ordered < vCPUs). +func TestAllocate_WholeCoreSMT_HybridSkipsSingleThread(t *testing.T) { + p := mustPlacer(t, hybridTopo(), 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r.Status != Success { + t.Fatalf("want Success, got %v (%s)", r.Status, r.Message) + } + a := r.Assignment + if len(a.OrderedHostCPUs) != 2 { + t.Fatalf("whole-core-smt must map one host CPU per vCPU (2), got %d: %v", + len(a.OrderedHostCPUs), a.OrderedHostCPUs) + } + if a.Guest.Threads != 2 { + t.Fatalf("guest threads must be 2, got %d", a.Guest.Threads) + } + c0 := p.topo.ByLCPU[a.OrderedHostCPUs[0]] + c1 := p.topo.ByLCPU[a.OrderedHostCPUs[1]] + if c0.Socket != c1.Socket || c0.CoreID != c1.CoreID { + t.Fatalf("vCPUs not on one physical core: %v", a.OrderedHostCPUs) + } + if len(c0.Siblings) != 2 { + t.Fatalf("whole-core-smt used a non-SMT core (siblings=%v)", c0.Siblings) + } +} + +// With both SMT cores unavailable and only single-thread E-cores free, +// whole-core-smt must fail cleanly with Insufficient rather than emit an +// assignment with fewer host CPUs than vCPUs. +func TestAllocate_WholeCoreSMT_OnlyECoresFree(t *testing.T) { + p := mustPlacer(t, hybridTopo(), 4) // reserve lcpu 0-3 -> both P-cores excluded + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r.Status != Insufficient { + t.Fatalf("no full-SMT core free must be Insufficient, got %v (%s)", r.Status, r.Message) + } +} + +// one-per-core may use single-thread E-cores (one vCPU per physical core, no +// sibling to park). +func TestAllocate_OnePerCore_UsesSingleThreadCores(t *testing.T) { + p := mustPlacer(t, hybridTopo(), 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeOnePerCore, NUMA: NUMALocal}) + if r.Status != Success { + t.Fatalf("want Success, got %v (%s)", r.Status, r.Message) + } + a := r.Assignment + if len(a.OrderedHostCPUs) != 4 { + t.Fatalf("one-per-core must map one host CPU per vCPU (4), got %v", a.OrderedHostCPUs) + } + usedECore := false + for _, c := range a.OrderedHostCPUs { + if len(p.topo.ByLCPU[c].Siblings) == 1 { + usedECore = true + } + } + if !usedECore { + t.Fatalf("one-per-core should be able to use single-thread cores: %v", a.OrderedHostCPUs) + } +} + +// A whole-core-smt VM and a one-per-core VM must coexist on a hybrid host with +// no physical core shared between them. +func TestAllocate_Hybrid_Coexistence(t *testing.T) { + p := mustPlacer(t, hybridTopo(), 0) + if r := p.Allocate(Request{UUID: u("smt"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}); r.Status != Success { + t.Fatalf("whole-core-smt vm should succeed, got %v (%s)", r.Status, r.Message) + } + if r := p.Allocate(Request{UUID: u("opc"), NumVCPUs: 4, Mode: ModeOnePerCore, NUMA: NUMALocal}); r.Status != Success { + t.Fatalf("one-per-core vm should coexist, got %v (%s)", r.Status, r.Message) + } + seen := map[cputopology.LCPU]bool{} + for _, c := range p.dedicated[u("smt")] { + seen[c] = true + } + for _, c := range p.dedicated[u("opc")] { + if seen[c] { + t.Fatalf("core sharing between VMs at lcpu %d", c) + } + } +} + +// best-effort stays within one NUMA node when the request fits. +func TestAllocate_BestEffort_FitsSingleNode(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}) + if r.Status != Success { + t.Fatalf("want Success, got %v (%s)", r.Status, r.Message) + } + if len(r.Assignment.NUMANodes) != 1 { + t.Fatalf("best-effort should stay on one node when it fits, got %v", r.Assignment.NUMANodes) + } +} + +// best-effort falls back to spanning nodes rather than failing when no single +// node fits (contrast NUMALocal, which returns NeedsRebalance here). +func TestAllocate_BestEffort_FallsBackToSpanning(t *testing.T) { + p := mustPlacer(t, twoNodesOneCoreEach(), 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}) + if r.Status != Success { + t.Fatalf("best-effort must fall back to spanning, got %v (%s)", r.Status, r.Message) + } + if len(r.Assignment.NUMANodes) != 2 { + t.Fatalf("expected spanning 2 nodes, got %v", r.Assignment.NUMANodes) + } +} + +// Reserve seeds a running VM's cores so a fresh Placer (post-restart) does not +// hand them to another VM; Free returns them to the pool. +func TestReserve_BlocksAndFrees(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) // 16 lcpus + if err := p.Reserve(u("running"), []uint32{0, 1, 2, 3}); err != nil { + t.Fatalf("Reserve of a running VM's cores: %v", err) + } + if len(p.FreeCPUs()) != 16-4 { + t.Fatalf("reserved cpus must be excluded from FreeCPUs, got %d", len(p.FreeCPUs())) + } + r := p.Allocate(Request{UUID: u("new"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) + if r.Status != Success { + t.Fatalf("new VM should allocate around reserved cores, got %v (%s)", r.Status, r.Message) + } + for _, c := range r.Assignment.OrderedHostCPUs { + if c <= 3 { + t.Fatalf("new VM got a reserved core %d", c) + } + } + if err := p.Reserve(u("running"), []uint32{0, 1, 2, 3}); err != nil { + t.Errorf("replaying the identical reservation must be a no-op, got %v", err) + } + p.Free(u("running")) + if len(p.FreeCPUs()) != 16-len(r.Assignment.OrderedHostCPUs) { + t.Fatalf("after Free, reserved cores must return to the free pool") + } +} + +// Reserve reseeds the placer from persisted DomainStatus, which is precisely +// where a stale or conflicting claim can appear. Recording one verbatim would let +// two workloads own a CPU, making HolderOf name an arbitrary owner and the +// allocator hand the CPU out twice, so every malformed claim must be refused -- +// and refused without changing anything. +func TestReserve_RejectsClaimsItCannotHonour(t *testing.T) { + running, other := u("running"), u("other") + + newSeeded := func(t *testing.T) *Placer { + t.Helper() + p := mustPlacer(t, twoSocketTopo(), 0) // lcpus 0..15 + if err := p.Reserve(running, []uint32{0, 1}); err != nil { + t.Fatalf("seed Reserve: %v", err) + } + return p + } + + tests := []struct { + name string + id uuid.UUID + cpus []uint32 + }{ + {"an empty CPU set claims nothing", other, nil}, + {"a logical CPU the host does not have", other, []uint32{99}}, + {"a CPU another workload already holds", other, []uint32{1}}, + {"a second, different claim by the same id", running, []uint32{2, 3}}, + {"a claim that only partly overlaps its own", running, []uint32{0, 1, 2}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p := newSeeded(t) + before := p.DedicatedSet() + if err := p.Reserve(tc.id, tc.cpus); err == nil { + t.Fatalf("Reserve(%s, %v) must be rejected", tc.id, tc.cpus) + } + if !equalLCPUs(before, p.DedicatedSet()) { + t.Errorf("a rejected Reserve must change nothing: %v -> %v", + before, p.DedicatedSet()) + } + }) + } + + // The same set in a different order is the same reservation, so replaying a + // status must not be treated as a conflict. + p := newSeeded(t) + if err := p.Reserve(running, []uint32{1, 0}); err != nil { + t.Errorf("re-reserving the same set in another order must be a no-op, got %v", err) + } +} + +// The zero value of Status must not be Success. A Result that was never filled +// in -- dropped in a map lookup miss, defaulted in a struct, decoded from an +// empty message -- would otherwise read as an approved placement, and a caller +// would pin a VM to an empty CPU set on the allocator's supposed authority. +func TestStatus_ZeroValueIsNotSuccess(t *testing.T) { + var missing Result + if missing.Status == Success { + t.Fatal("the zero Status must not be Success") + } + if missing.Status != StatusUnspecified { + t.Fatalf("the zero Status must be StatusUnspecified, got %v", missing.Status) + } + if got := StatusUnspecified.String(); got != "unspecified" { + t.Errorf("StatusUnspecified.String() = %q", got) + } + // A map miss is the way this reaches a caller in practice. + if plan := map[uuid.UUID]Result{}; plan[u("never-planned")].Status == Success { + t.Error("a workload absent from a plan must not read as placed") + } +} + +// noSMTTopo is a node whose cores have a single hardware thread each: SMT +// disabled in firmware, or a part that never had it (most ARM64). +func noSMTTopo() *cputopology.Topology { + var infos []cputopology.CoreInfo + for core := uint(0); core < 4; core++ { + infos = append(infos, cputopology.CoreInfo{LCore: core, CoreID: core}) + } + return cputopology.BuildTopology(infos) +} + +// A whole-core-SMT request on a node with no SMT-capable core at all is +// unsatisfiable, not short of capacity. Reported as a shortage it produced advice +// -- stop another pinned workload, get a node with more cores -- of which every +// clause is false here, sending the operator after capacity that could never +// help. This is the likeliest failure of the default pinning policy. +func TestAllocate_WholeCoreSMT_NoSMTAnywhereIsUnsatisfiable(t *testing.T) { + p := mustPlacer(t, noSMTTopo(), 0) + r := p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}) + if r.Status != InvalidRequest { + t.Fatalf("want InvalidRequest, got %v (%s)", r.Status, r.Message) + } + if !r.TopologyUnsupported { + t.Error("the node cannot satisfy this in any arrangement; TopologyUnsupported must be set") + } + if r.CoresNeeded != 0 || r.CoresFree != 0 { + t.Errorf("an unsatisfiable request is not a shortage, so no core counts: got %d/%d", + r.CoresNeeded, r.CoresFree) + } + for _, want := range []string{"SMT sibling", "threads=2"} { + if !strings.Contains(r.Message, want) { + t.Errorf("message must say what is wrong (%q), got %q", want, r.Message) + } + } + // Nothing was reserved, and the node is still usable for one-per-core. + if len(p.DedicatedSet()) != 0 { + t.Errorf("a refused request must reserve nothing, got %v", p.DedicatedSet()) + } + if opc := p.Allocate(Request{UUID: u("opc"), NumVCPUs: 2, Mode: ModeOnePerCore, + NUMA: NUMABestEffort}); opc.Status != Success { + t.Errorf("one-per-core must still work on a non-SMT node, got %v (%s)", + opc.Status, opc.Message) + } +} + +// The contrast case: SMT-capable cores do exist but are all taken. That IS a +// shortage -- freeing CPUs would help -- so it must stay Insufficient with the +// core counts the retry condition quotes, and must not claim the topology cannot +// do it. +func TestAllocate_WholeCoreSMT_BusySMTCoresAreAShortage(t *testing.T) { + p := mustPlacer(t, twoCoresSMT2(), 0) // cores {0,4} and {1,5} + if r := p.Allocate(Request{UUID: u("holder"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, + NUMA: NUMABestEffort}); r.Status != Success { + t.Fatalf("holder must take both cores, got %v (%s)", r.Status, r.Message) + } + r := p.Allocate(Request{UUID: u("late"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}) + if r.Status != Insufficient { + t.Fatalf("want Insufficient, got %v (%s)", r.Status, r.Message) + } + if r.TopologyUnsupported { + t.Error("the cores exist and are merely busy; freeing them would help") + } + if r.CoresNeeded != 1 || r.CoresFree != 0 { + t.Errorf("want CoresNeeded=1 CoresFree=0, got %d/%d", r.CoresNeeded, r.CoresFree) + } +} + +// The SMT sibling parked by a one-per-core request is held, not idle capacity: +// handing it to anyone else gives back exactly the interference the mode exists +// to remove. Every view of the node must agree it is taken. +func TestAllocate_OnePerCore_ParkedSiblingIsWithheldEverywhere(t *testing.T) { + p := mustPlacer(t, twoCoresSMT2(), 0) // cores {0,4} and {1,5} + r := p.Allocate(Request{UUID: u("opc"), NumVCPUs: 1, Mode: ModeOnePerCore, NUMA: NUMABestEffort}) + if r.Status != Success { + t.Fatalf("want Success, got %v (%s)", r.Status, r.Message) + } + if len(r.Assignment.ParkedCPUs) != 1 { + t.Fatalf("want exactly one parked sibling, got %v", r.Assignment.ParkedCPUs) + } + parked := r.Assignment.ParkedCPUs[0] + + for _, c := range p.FreeCPUs() { + if c == parked { + t.Errorf("parked sibling %d must not be free, got %v", parked, p.FreeCPUs()) + } + } + shared, err := p.AllocateShared(u("legacy"), 2) + if err != nil { + t.Fatalf("the remaining whole core must still serve a shared request: %v", err) + } + for _, c := range shared { + if c == parked { + t.Errorf("shared allocation took the parked sibling %d: %v", parked, shared) + } + } + dedicated := poolByKind(t, p.PoolUtilization(nil), PoolDedicated) + found := false + for _, c := range dedicated.CPUs { + if c == parked { + found = true + } + } + if !found { + t.Errorf("parked sibling %d must be reported in the dedicated pool, got %v", + parked, dedicated.CPUs) + } + if dedicated.FreeThreads != 0 { + t.Errorf("nothing in the dedicated pool is free, got %d free threads", + dedicated.FreeThreads) + } +} + +// The guest -smp topology and the host CPU list are two views of one decision and +// nothing downstream cross-checks them: an inconsistency reaches the operator as +// a domain that will not start. validate is the only place that can catch it, so +// each way of getting it wrong must be caught. +func TestAssignment_ValidateRejectsInconsistentPlacements(t *testing.T) { + topo := twoNodesOneCoreEach() // core {0,1} in node 0, core {2,3} in node 1 + + good := &Assignment{ + OrderedHostCPUs: lcpus(0, 1), + Guest: GuestTopology{Sockets: 1, Cores: 1, Threads: 2}, + NUMANodes: []uint{0}, + } + if err := good.validate(topo); err != nil { + t.Fatalf("a consistent assignment must validate: %v", err) + } + + tests := []struct { + name string + a *Assignment + }{ + {"vCPU count disagrees with the guest topology", &Assignment{ + OrderedHostCPUs: lcpus(0, 1, 2), + Guest: GuestTopology{Sockets: 1, Cores: 1, Threads: 2}, + NUMANodes: []uint{0, 1}, + }}, + {"one host CPU serving two vCPUs", &Assignment{ + OrderedHostCPUs: lcpus(0, 0), + Guest: GuestTopology{Sockets: 1, Cores: 1, Threads: 2}, + NUMANodes: []uint{0}, + }}, + {"a CPU both parked and running a vCPU", &Assignment{ + OrderedHostCPUs: lcpus(0, 1), + Guest: GuestTopology{Sockets: 1, Cores: 1, Threads: 2}, + ParkedCPUs: lcpus(1), + NUMANodes: []uint{0}, + }}, + {"a host CPU the topology does not have", &Assignment{ + OrderedHostCPUs: lcpus(0, 99), + Guest: GuestTopology{Sockets: 1, Cores: 1, Threads: 2}, + NUMANodes: []uint{0}, + }}, + {"NUMANodes missing a node the CPUs are in", &Assignment{ + OrderedHostCPUs: lcpus(0, 2), + Guest: GuestTopology{Sockets: 1, Cores: 2, Threads: 1}, + NUMANodes: []uint{0}, + }}, + {"NUMANodes naming a node no CPU is in", &Assignment{ + OrderedHostCPUs: lcpus(0, 1), + Guest: GuestTopology{Sockets: 1, Cores: 1, Threads: 2}, + NUMANodes: []uint{1}, + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if err := tc.a.validate(topo); err == nil { + t.Errorf("must be rejected: %+v", tc.a) + } + }) + } +} + +// Every assignment Allocate returns must pass its own consistency check, across +// the modes and NUMA policies that build one differently. +func TestAllocate_SuccessfulAssignmentsAreConsistent(t *testing.T) { + for _, tc := range []struct { + name string + topo *cputopology.Topology + r Request + }{ + {"whole-core-smt in one node", twoSocketTopo(), + Request{UUID: u("a"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}}, + {"whole-core-smt spanning nodes", twoNodesOneCoreEach(), + Request{UUID: u("b"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMAAllowCross}}, + {"one-per-core parking siblings", twoSocketTopo(), + Request{UUID: u("c"), NumVCPUs: 3, Mode: ModeOnePerCore, NUMA: NUMABestEffort}}, + {"one-per-core on single-thread cores", hybridTopo(), + Request{UUID: u("d"), NumVCPUs: 5, Mode: ModeOnePerCore, NUMA: NUMABestEffort}}, + } { + t.Run(tc.name, func(t *testing.T) { + p := mustPlacer(t, tc.topo, 0) + res := p.Allocate(tc.r) + if res.Status != Success { + t.Fatalf("want Success, got %v (%s)", res.Status, res.Message) + } + if err := res.Assignment.validate(p.topo); err != nil { + t.Errorf("Allocate returned an inconsistent assignment: %v", err) + } + }) + } +} + +func TestFreeCPUs_ExcludesBoth(t *testing.T) { + p := mustPlacer(t, twoSocketTopo(), 0) // 16 lcpus total + _ = p.Allocate(Request{UUID: u("vm1"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}) // 4 lcpus + _, _ = p.AllocateShared(u("legacy"), 2) // 2 lcpus + free := p.FreeCPUs() + if len(free) != 16-4-2 { + t.Fatalf("FreeCPUs should exclude both allocations: got %d", len(free)) + } + p.Free(u("vm1")) + p.Free(u("legacy")) + if len(p.FreeCPUs()) != 16 { + t.Fatalf("after Free all cpus should be free") + } +} diff --git a/pkg/pillar/cpuallocator/plan.go b/pkg/pillar/cpuallocator/plan.go new file mode 100644 index 00000000000..be4763414cc --- /dev/null +++ b/pkg/pillar/cpuallocator/plan.go @@ -0,0 +1,183 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cpuallocator + +import ( + "fmt" + "math" + "sort" + + uuid "github.com/satori/go.uuid" +) + +// Plan computes placement for a whole set of workloads at once and returns one +// result per workload. +// +// Planning the set together, rather than allocating for each workload as it +// happens to activate, is what makes placement independent of order. Allocating +// incrementally meant whichever workload activated first won the scarce cores: +// a flexible workload could take the only SMT-capable core, leaving a workload +// that *needs* one unplaceable, and the same set of apps could land differently +// on each boot. Here the requests are ordered by how constrained they are -- +// the workloads with the fewest possible placements first -- so the outcome +// depends only on the set itself. +// +// Plan does not mutate the placer: it neither reserves nor frees anything. The +// caller applies a planned assignment with Reserve when the workload actually +// starts, which is what lets a workload that has not started yet -- or starts +// late -- still claim the CPUs the plan set aside for it. +func (p *Placer) Plan(requests []Request) map[uuid.UUID]Result { + ordered := make([]Request, len(requests)) + copy(ordered, requests) + sort.Slice(ordered, func(i, j int) bool { + if a, b := constraintRank(ordered[i]), constraintRank(ordered[j]); a != b { + return a < b + } + // Among equally constrained workloads the larger one is harder to fit, + // so place it while there is still room. + if ordered[i].NumVCPUs != ordered[j].NumVCPUs { + return ordered[i].NumVCPUs > ordered[j].NumVCPUs + } + // Tie-break on identity so the order is total and therefore stable. + return ordered[i].UUID.String() < ordered[j].UUID.String() + }) + + // Plan from an empty slate so the result is a function of the request set + // alone, not of whatever is allocated at this moment. + scratch := newPlacer(p.topo, p.numReservedForEVE) + plan := make(map[uuid.UUID]Result, len(ordered)) + for _, request := range ordered { + // One unplaceable workload must not stop the rest from being placed. + plan[request.UUID] = scratch.place(request) + } + return plan +} + +// place runs one planned request, routing thread-granular ones to the shared +// allocator. Those are planned as well, even though nothing applies a planned +// shared assignment, because they still take CPUs exclusively: a caller +// deriving a CPU set that no pinned workload will ever occupy has to know about +// them too. +func (p *Placer) place(r Request) Result { + if r.Mode != ModeShared { + return p.Allocate(r) + } + cpus, err := p.AllocateShared(r.UUID, r.NumVCPUs) + if err != nil { + return Result{Status: Insufficient, Message: err.Error()} + } + return Result{Status: Success, Assignment: &Assignment{OrderedHostCPUs: cpus}} +} + +// constraintRank orders workloads from fewest possible placements to most. +// whole-core-SMT is the most constrained: it can only use a core that actually +// has two hardware threads, and on a hybrid or SMT-disabled machine most cores +// do not. one-per-core will take any physical core. Anything else is +// thread-granular and can go almost anywhere. +func constraintRank(r Request) int { + switch r.Mode { + case ModeWholeCoreSMT: + return 0 + case ModeOnePerCore: + return 1 + default: + return 2 + } +} + +// String implements fmt.Stringer so an outcome can be reported in a status or a +// diagnostic without every caller mapping the values itself. +func (s Status) String() string { + switch s { + case StatusUnspecified: + return "unspecified" + case Success: + return "success" + case NeedsRebalance: + return "needs-rebalance" + case Insufficient: + return "insufficient" + case InvalidRequest: + return "invalid-request" + } + return fmt.Sprintf("unknown(%d)", int(s)) +} + +// Score ranks an assignment by placement quality, lower being better. It is +// compared lexicographically: unrecognised CPUs dominate, then NUMA locality, +// then cache locality. +// +// Deliberately absent is which CPU indices were used. Many assignments share the +// best score -- any whole core in the right NUMA node and L3 domain is as good as +// any other -- so a placement that differs only in indices is not worse. Treating +// index choice as quality would make a workload look mis-placed simply because +// its first-choice CPUs were taken, and would demand pointless restarts. +type Score struct { + // UnknownCPUs is how many of the assignment's host CPUs the topology does + // not have. It dominates because such an assignment cannot be judged at all: + // skipping the CPUs silently scored a wholly bogus assignment as perfect + // {0,0}, which made a healthy live placement compare as worse than a bogus + // plan and produced a repack recommendation derived from nothing. + UnknownCPUs int + // NUMANodes is how many NUMA nodes the assignment spans. Spanning nodes + // costs cross-socket memory latency on every access. + NUMANodes int + // L3Domains is how many last-level caches it spans. + L3Domains int +} + +// WorseThan reports whether this score is strictly worse than another. +func (s Score) WorseThan(other Score) bool { + if s.UnknownCPUs != other.UnknownCPUs { + return s.UnknownCPUs > other.UnknownCPUs + } + if s.NUMANodes != other.NUMANodes { + return s.NUMANodes > other.NUMANodes + } + return s.L3Domains > other.L3Domains +} + +// worstScore is the score of something that cannot be evaluated at all. It is +// worse than every real score, so a comparison against it can only ever conclude +// "the alternative is at least as good". +var worstScore = Score{UnknownCPUs: math.MaxInt, NUMANodes: math.MaxInt, L3Domains: math.MaxInt} + +// Score computes the quality of an assignment against the topology. +// +// A nil assignment scores the worst possible value, not the best. "Nothing is +// placed" must never compare as at least as good as a real placement, or a +// workload with no assignment would silently pass an is-this-good-enough check +// instead of being reported as unplaced. +func (p *Placer) Score(a *Assignment) Score { + if a == nil { + return worstScore + } + unknown := 0 + numaNodes := map[uint]bool{} + l3Domains := map[uint]bool{} + // A core whose L3 id the kernel does not expose counts as a cache domain of + // its own. Grouping such cores under their placeholder id would report a + // workload split across several last-level caches as perfectly cache-local, + // which is the one direction this score must never err in. + type physicalCore struct{ socket, core uint } + unknownL3 := map[physicalCore]bool{} + for _, cpu := range a.OrderedHostCPUs { + core, known := p.topo.ByLCPU[cpu] + if !known { + unknown++ + continue + } + numaNodes[core.NUMA] = true + if core.L3Unknown { + unknownL3[physicalCore{core.Socket, core.CoreID}] = true + } else { + l3Domains[core.L3ID] = true + } + } + return Score{ + UnknownCPUs: unknown, + NUMANodes: len(numaNodes), + L3Domains: len(l3Domains) + len(unknownL3), + } +} diff --git a/pkg/pillar/cpuallocator/plan_test.go b/pkg/pillar/cpuallocator/plan_test.go new file mode 100644 index 00000000000..bdb7ef46ade --- /dev/null +++ b/pkg/pillar/cpuallocator/plan_test.go @@ -0,0 +1,427 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cpuallocator + +import ( + "fmt" + "testing" + + uuid "github.com/satori/go.uuid" + + "github.com/lf-edge/eve/pkg/pillar/cputopology" +) + +// permutationsOf returns every ordering of the requests. A single reversal is not +// enough to pin order-independence down: it exercises one of n! arrival orders, +// and the orders that expose a partial sort key are usually not that one. +func permutationsOf(requests []Request) [][]Request { + if len(requests) <= 1 { + return [][]Request{append([]Request{}, requests...)} + } + var out [][]Request + for i := range requests { + rest := make([]Request, 0, len(requests)-1) + rest = append(rest, requests[:i]...) + rest = append(rest, requests[i+1:]...) + for _, tail := range permutationsOf(rest) { + out = append(out, append([]Request{requests[i]}, tail...)) + } + } + return out +} + +// samePlan reports the first difference between two plans, or "" if they agree. +func samePlan(want, got map[uuid.UUID]Result) string { + if len(want) != len(got) { + return fmt.Sprintf("plans differ in size: %d vs %d", len(want), len(got)) + } + for id, w := range want { + g, ok := got[id] + if !ok { + return fmt.Sprintf("workload %s missing", id) + } + if w.Status != g.Status { + return fmt.Sprintf("%s: status %v vs %v", id, w.Status, g.Status) + } + if w.Status != Success { + continue + } + if !equalLCPUs(w.Assignment.OrderedHostCPUs, g.Assignment.OrderedHostCPUs) { + return fmt.Sprintf("%s: vCPUs %v vs %v", id, + w.Assignment.OrderedHostCPUs, g.Assignment.OrderedHostCPUs) + } + if !equalLCPUs(w.Assignment.ParkedCPUs, g.Assignment.ParkedCPUs) { + return fmt.Sprintf("%s: parked %v vs %v", id, + w.Assignment.ParkedCPUs, g.Assignment.ParkedCPUs) + } + } + return "" +} + +// Planning the same set of workloads must produce the same assignment whatever +// order the requests arrive in. This is the property that removes the boot-order +// race: allocating incrementally, whoever activated first won the scarce cores. +func TestPlan_IsOrderIndependent(t *testing.T) { + requests := []Request{ + {UUID: u("smt"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}, + {UUID: u("ope"), NumVCPUs: 2, Mode: ModeOnePerCore, NUMA: NUMABestEffort}, + {UUID: u("smt2"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}, + } + + orders := permutationsOf(requests) + reference := mustPlacer(t, twoSocketTopo(), 0).Plan(orders[0]) + for _, order := range orders[1:] { + got := mustPlacer(t, twoSocketTopo(), 0).Plan(order) + if diff := samePlan(reference, got); diff != "" { + ids := make([]string, 0, len(order)) + for _, r := range order { + ids = append(ids, r.UUID.String()) + } + t.Errorf("plan depends on arrival order %v: %s", ids, diff) + } + } +} + +// Two workloads identical in mode and size are ordered by nothing but their +// identity, and the sort's third key is the only thing that makes that order +// total. Without it Go's unstable sort leaves them in demand-set order, so two +// equally shaped apps swap NUMA nodes depending on which status arrived first -- +// a different layout after every reboot, for no reason the operator can see. +func TestPlan_EquallyShapedWorkloadsAreOrderedByIdentity(t *testing.T) { + // One core per NUMA node, so only one of the two can have node 0 and the + // choice between them is decided purely by the tie-break. + requests := []Request{ + {UUID: u("twin-a"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}, + {UUID: u("twin-b"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}, + } + forward := mustPlacer(t, twoNodesOneCoreEach(), 0).Plan(requests) + backward := mustPlacer(t, twoNodesOneCoreEach(), 0).Plan( + []Request{requests[1], requests[0]}) + + for _, id := range []uuid.UUID{requests[0].UUID, requests[1].UUID} { + if forward[id].Status != Success { + t.Fatalf("%s: want Success, got %v (%s)", id, forward[id].Status, forward[id].Message) + } + } + if diff := samePlan(forward, backward); diff != "" { + t.Errorf("equally shaped workloads swapped places with the arrival order: %s", diff) + } + // They must genuinely be competing for one node, or the tie-break is untested. + nodesA := forward[requests[0].UUID].Assignment.NUMANodes + nodesB := forward[requests[1].UUID].Assignment.NUMANodes + if len(nodesA) != 1 || len(nodesB) != 1 || nodesA[0] == nodesB[0] { + t.Fatalf("the twins must contend for one node, got %v and %v", nodesA, nodesB) + } +} + +// The tightest-constrained workload must be placed first, so a flexible one +// cannot take the only core that the constrained one could have used. On a +// topology with a single SMT-capable core, an interleaved plan that served the +// one-per-core request first would leave whole-core-SMT unsatisfiable. +func TestPlan_TightestConstraintFirst(t *testing.T) { + // One SMT core (2 threads) plus two single-thread cores. + infos := []cputopology.CoreInfo{ + {LCore: 0, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, + {LCore: 1, Socket: 0, CoreID: 0, NUMA: 0, L3ID: 0}, // sibling of 0 + {LCore: 2, Socket: 0, CoreID: 1, NUMA: 0, L3ID: 0}, + {LCore: 3, Socket: 0, CoreID: 2, NUMA: 0, L3ID: 0}, + } + plan := mustPlacer(t, cputopology.BuildTopology(infos), 0).Plan([]Request{ + // Deliberately listed with the flexible request first. + {UUID: u("flexible"), NumVCPUs: 1, Mode: ModeOnePerCore, NUMA: NUMABestEffort}, + {UUID: u("needs-smt"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}, + }) + + smt := plan[u("needs-smt")] + if smt.Status != Success { + t.Fatalf("whole-core-smt must be placed first and succeed, got %v (%s)", + smt.Status, smt.Message) + } + // It must have taken the one two-thread core. + if !equalLCPUs(smt.Assignment.OrderedHostCPUs, []cputopology.LCPU{0, 1}) { + t.Errorf("whole-core-smt got %v, want the SMT core [0 1]", + smt.Assignment.OrderedHostCPUs) + } + if flexible := plan[u("flexible")]; flexible.Status != Success { + t.Errorf("the flexible workload should still fit on a single-thread core, got %v (%s)", + flexible.Status, flexible.Message) + } +} + +// Planned workloads must never be given overlapping CPUs, including the +// siblings parked by a one-per-core request. +func TestPlan_AssignmentsAreDisjoint(t *testing.T) { + plan := mustPlacer(t, twoSocketTopo(), 0).Plan([]Request{ + {UUID: u("a"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}, + {UUID: u("b"), NumVCPUs: 2, Mode: ModeOnePerCore, NUMA: NUMABestEffort}, + {UUID: u("c"), NumVCPUs: 2, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}, + }) + + owner := map[cputopology.LCPU]uuid.UUID{} + for id, result := range plan { + if result.Status != Success { + continue + } + all := append(append([]cputopology.LCPU{}, result.Assignment.OrderedHostCPUs...), + result.Assignment.ParkedCPUs...) + for _, cpu := range all { + if other, taken := owner[cpu]; taken { + t.Fatalf("CPU %d assigned to both %s and %s", cpu, other, id) + } + owner[cpu] = id + } + } +} + +// A workload that cannot be placed must be reported as such without preventing +// the others from being placed: one unsatisfiable request is not a reason to +// leave the whole node unallocated. +func TestPlan_UnsatisfiableRequestDoesNotBlockOthers(t *testing.T) { + plan := mustPlacer(t, twoSocketTopo(), 0).Plan([]Request{ + {UUID: u("fits"), NumVCPUs: 2, Mode: ModeOnePerCore, NUMA: NUMABestEffort}, + {UUID: u("huge"), NumVCPUs: 64, Mode: ModeOnePerCore, NUMA: NUMABestEffort}, + }) + + if got := plan[u("fits")].Status; got != Success { + t.Errorf("the satisfiable request must still be placed, got %v", got) + } + if got := plan[u("huge")].Status; got == Success { + t.Error("a request for more cores than exist must not succeed") + } +} + +// applyPlan reserves every successfully planned assignment, as domainmgr does +// when the workloads actually start. Reserve rejecting anything would itself mean +// the plan handed one CPU to two workloads. +func applyPlan(t *testing.T, placer *Placer, plan map[uuid.UUID]Result) { + t.Helper() + for id, result := range plan { + if result.Status != Success { + continue + } + var cpus []uint32 + for _, c := range result.Assignment.OrderedHostCPUs { + cpus = append(cpus, uint32(c)) + } + for _, c := range result.Assignment.ParkedCPUs { + cpus = append(cpus, uint32(c)) + } + if err := placer.Reserve(id, cpus); err != nil { + t.Fatalf("applying the plan for %s: %v", id, err) + } + } +} + +// Adding a workload must not move the ones already running. Re-planning starts +// from an empty slate, so nothing stops the allocator from producing a better +// global layout that relocates a running VM -- and relocating it would mean +// restarting it. What prevents that is the ordering: the new workload is less +// constrained than the running ones, so it is placed after them and takes only +// what is left. This test fails if that ordering is weakened. +func TestPlan_AddingAWorkloadDoesNotMoveRunningOnes(t *testing.T) { + running := []Request{ + {UUID: u("a"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMABestEffort}, + {UUID: u("b"), NumVCPUs: 2, Mode: ModeOnePerCore, NUMA: NUMABestEffort}, + } + placer := mustPlacer(t, twoSocketTopo(), 0) + before := placer.Plan(running) + applyPlan(t, placer, before) + + newcomer := Request{UUID: u("c"), NumVCPUs: 3, Mode: ModeShared} + after := placer.Plan(append(append([]Request{}, running...), newcomer)) + + for _, r := range running { + want, got := before[r.UUID], after[r.UUID] + if want.Status != Success { + t.Fatalf("%s was not placed to begin with: %v (%s)", r.UUID, want.Status, want.Message) + } + if got.Status != Success { + t.Errorf("%s lost its placement when %s appeared: %v (%s)", + r.UUID, newcomer.UUID, got.Status, got.Message) + continue + } + if !equalLCPUs(want.Assignment.OrderedHostCPUs, got.Assignment.OrderedHostCPUs) { + t.Errorf("%s would have to be restarted: vCPUs moved %v -> %v", r.UUID, + want.Assignment.OrderedHostCPUs, got.Assignment.OrderedHostCPUs) + } + if !equalLCPUs(want.Assignment.ParkedCPUs, got.Assignment.ParkedCPUs) { + t.Errorf("%s: parked siblings moved %v -> %v", r.UUID, + want.Assignment.ParkedCPUs, got.Assignment.ParkedCPUs) + } + } + if c := after[newcomer.UUID]; c.Status != Success { + t.Errorf("the new workload must still be placed, got %v (%s)", c.Status, c.Message) + } +} + +// A thread-granular shortage must not report core counts. domainmgr decides +// whether a refusal can clear by freeing whole cores from these numbers, and a +// shared request is not measured in cores at all -- a nonzero count here would +// send it down the wrong retry path. +func TestPlan_SharedShortageReportsNoCoreCounts(t *testing.T) { + plan := mustPlacer(t, twoCoresSMT2(), 0).Plan([]Request{ + {UUID: u("greedy"), NumVCPUs: 64, Mode: ModeShared}, + }) + result := plan[u("greedy")] + if result.Status != Insufficient { + t.Fatalf("want Insufficient, got %v (%s)", result.Status, result.Message) + } + if result.CoresNeeded != 0 || result.CoresFree != 0 { + t.Errorf("a thread-granular shortage is not counted in cores, got %d/%d", + result.CoresNeeded, result.CoresFree) + } +} + +// Reserved CPUs backing EVE's own housekeeping must stay out of every plan. +func TestPlan_HonorsReservedCPUs(t *testing.T) { + const reserved = 2 + plan := mustPlacer(t, twoSocketTopo(), reserved).Plan([]Request{ + {UUID: u("a"), NumVCPUs: 2, Mode: ModeOnePerCore, NUMA: NUMABestEffort}, + }) + result := plan[u("a")] + if result.Status != Success { + t.Fatalf("want Success, got %v (%s)", result.Status, result.Message) + } + for _, cpu := range result.Assignment.OrderedHostCPUs { + if uint32(cpu) < reserved { + t.Errorf("plan used reserved CPU %d", cpu) + } + } +} + +// Thread-granular workloads must appear in the plan too: they take CPUs +// exclusively, so a caller deriving a housekeeping set from the plan has to see +// them, and they must never be planned onto a whole-core workload's cores. +func TestPlan_IncludesThreadGranularWorkloads(t *testing.T) { + plan := mustPlacer(t, twoSocketTopo(), 0).Plan([]Request{ + {UUID: u("wholecore"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}, + {UUID: u("legacy"), NumVCPUs: 3, Mode: ModeShared}, + }) + + legacy := plan[u("legacy")] + if legacy.Status != Success || len(legacy.Assignment.OrderedHostCPUs) != 3 { + t.Fatalf("thread-granular workload must be planned, got %v (%s)", + legacy.Status, legacy.Message) + } + whole := plan[u("wholecore")] + if whole.Status != Success { + t.Fatalf("whole-core workload must still be planned, got %v (%s)", + whole.Status, whole.Message) + } + taken := map[cputopology.LCPU]bool{} + for _, cpu := range whole.Assignment.OrderedHostCPUs { + taken[cpu] = true + } + for _, cpu := range legacy.Assignment.OrderedHostCPUs { + if taken[cpu] { + t.Errorf("thread-granular plan overlaps a whole-core one at cpu %d", cpu) + } + } +} + +func equalLCPUs(a, b []cputopology.LCPU) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestScore_WorseThan(t *testing.T) { + // NUMA locality dominates cache locality. + oneNodeTwoL3 := Score{NUMANodes: 1, L3Domains: 2} + twoNodesOneL3 := Score{NUMANodes: 2, L3Domains: 1} + if !twoNodesOneL3.WorseThan(oneNodeTwoL3) { + t.Error("spanning two NUMA nodes must be worse than spanning two L3 domains") + } + if oneNodeTwoL3.WorseThan(twoNodesOneL3) { + t.Error("comparison must not be symmetric") + } + // Equal scores are not worse -- this is what stops a placement that merely + // used different CPU indices from being reported as sub-optimal. + equal := Score{NUMANodes: 1, L3Domains: 1} + if equal.WorseThan(equal) { + t.Error("an equal score must not count as worse") + } + // An unrecognised CPU dominates every locality difference: an assignment + // nobody can interpret must not win a comparison against one that is merely + // spread out. + bogus := Score{UnknownCPUs: 1} + if !bogus.WorseThan(Score{NUMANodes: 4, L3Domains: 4}) { + t.Error("an unrecognised CPU must outweigh any amount of poor locality") + } + if (Score{NUMANodes: 4, L3Domains: 4}).WorseThan(bogus) { + t.Error("poor locality must not be reported as worse than an uninterpretable CPU") + } +} + +func TestScore_CountsSpannedDomains(t *testing.T) { + placer := mustPlacer(t, twoSocketTopo(), 0) + // twoSocketTopo puts each socket in its own NUMA node and L3 domain, so a + // request that fits in one socket must score 1/1. + local := placer.Plan([]Request{ + {UUID: u("local"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMALocal}, + })[u("local")] + if local.Status != Success { + t.Fatalf("want Success, got %v (%s)", local.Status, local.Message) + } + if got := placer.Score(local.Assignment); got != (Score{NUMANodes: 1, L3Domains: 1}) { + t.Errorf("NUMA-local assignment scored %+v, want 1 node / 1 L3", got) + } +} + +// A placement forced to span the machine must be scored as spanning it: the +// spread is what a sub-optimal-placement report exists to surface, so the counts +// have to grow with it rather than saturate at 1. +func TestScore_CountsEveryDomainSpanned(t *testing.T) { + placer := mustPlacer(t, twoNodesOneCoreEach(), 0) // one core per node, own L3 each + spanning := placer.Plan([]Request{ + {UUID: u("spanning"), NumVCPUs: 4, Mode: ModeWholeCoreSMT, NUMA: NUMAAllowCross}, + })[u("spanning")] + if spanning.Status != Success { + t.Fatalf("want Success, got %v (%s)", spanning.Status, spanning.Message) + } + want := Score{NUMANodes: 2, L3Domains: 2} + if got := placer.Score(spanning.Assignment); got != want { + t.Errorf("spanning assignment scored %+v, want %+v", got, want) + } + if !placer.Score(spanning.Assignment).WorseThan(Score{NUMANodes: 1, L3Domains: 1}) { + t.Error("spanning both nodes must score worse than fitting in one") + } +} + +// Two things Score must never call good: CPUs the topology does not know, and no +// assignment at all. Ignoring unknown CPUs scored a wholly bogus assignment as +// perfect, so a bogus plan made a healthy live placement look sub-optimal and +// demanded a repack; and a nil assignment scoring zero meant an unplaced workload +// silently passed for well placed. +func TestScore_UnknownAndMissingAssignmentsAreNotOptimal(t *testing.T) { + placer := mustPlacer(t, twoSocketTopo(), 0) + healthy := Score{NUMANodes: 1, L3Domains: 1} + + bogus := placer.Score(&Assignment{OrderedHostCPUs: lcpus(200, 201)}) + if bogus.UnknownCPUs != 2 { + t.Errorf("want both unknown CPUs counted, got %+v", bogus) + } + if !bogus.WorseThan(healthy) { + t.Errorf("an assignment of CPUs this host does not have must be worse than a real one, got %+v", bogus) + } + if healthy.WorseThan(bogus) { + t.Error("a healthy placement must not be reported as worse than a bogus one") + } + + missing := placer.Score(nil) + if !missing.WorseThan(healthy) { + t.Errorf("no assignment must score worse than any real one, got %+v", missing) + } + // Nothing may be worse than "not placed", including itself: the comparison + // must never conclude a real placement should be replaced by nothing. + if healthy.WorseThan(missing) || missing.WorseThan(missing) { + t.Error("nothing may be reported as worse than a missing assignment") + } +} diff --git a/pkg/pillar/cpuallocator/pools.go b/pkg/pillar/cpuallocator/pools.go new file mode 100644 index 00000000000..624960152d1 --- /dev/null +++ b/pkg/pillar/cpuallocator/pools.go @@ -0,0 +1,157 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cpuallocator + +import ( + "fmt" + "sort" + + "github.com/lf-edge/eve/pkg/pillar/cputopology" +) + +// CPUPool names one partition of the node's logical CPUs. +type CPUPool int + +const ( + // PoolHousekeeping is every logical CPU no workload holds exclusively: the + // CPUs reserved for EVE itself plus whatever is left over, which is where + // every workload that did not ask for dedicated placement runs. + PoolHousekeeping CPUPool = iota + // PoolDedicated is every logical CPU held exclusively by some workload, + // including SMT siblings parked idle by a one-thread-per-core request. + PoolDedicated + // PoolIsolated is the set the running kernel isolates (isolcpus). Unlike the + // other two it is not part of the partition: it is a kernel fact that cuts + // across both, reported so a consumer can see how much shielded capacity + // exists and how much of it is already taken. + PoolIsolated +) + +// String implements fmt.Stringer. +func (p CPUPool) String() string { + switch p { + case PoolHousekeeping: + return "housekeeping" + case PoolDedicated: + return "dedicated" + case PoolIsolated: + return "isolated" + default: + return fmt.Sprintf("CPUPool(%d)", int(p)) + } +} + +// PoolUtilization is one pool's extent and how much of it is still available. +// +// Both the sets and the counts are reported because they answer different +// questions, and the whole-core counts are not derivable from a thread count. +// A thread sitting on a core whose sibling is taken cannot be handed to a +// workload that asked for whole physical cores, so "how many threads are free" +// and "how many cores can still be given out whole" are genuinely different +// numbers -- see FreeWholeCores. +type PoolUtilization struct { + Pool CPUPool + // CPUs is every logical CPU in this pool, ascending. + CPUs []cputopology.LCPU + // FreeCPUs is the subset of CPUs that could still be handed to a workload + // asking for dedicated placement: not held by any workload, and not part of + // the range reserved for EVE. + FreeCPUs []cputopology.LCPU + // TotalThreads, AllocatedThreads and FreeThreads summarise the sets above. + TotalThreads uint32 + AllocatedThreads uint32 + FreeThreads uint32 + // TotalCores counts the physical cores all of whose SMT siblings are in + // this pool. A core straddling two pools counts towards neither. + TotalCores uint32 + // FreeWholeCores counts the physical cores all of whose SMT siblings are + // free. This -- not FreeThreads -- is what bounds how many more whole-core + // workloads the node can take. + FreeWholeCores uint32 +} + +// PoolUtilization reports every CPU pool of the node and how much of each is +// left, for the node-level "will it fit?" report. +// +// isolated is the set the running kernel isolates, which the allocator does not +// discover for itself: it is a kernel fact read from sysfs by the caller. Ids +// the topology does not know are ignored. +func (p *Placer) PoolUtilization(isolated []cputopology.LCPU) []PoolUtilization { + p.mu.Lock() + defer p.mu.Unlock() + + dedicated := p.dedicatedLookup() + // free means "could still be given to a workload asking for dedicated + // placement": the same two exclusions Allocate applies. + isFree := func(c cputopology.LCPU) bool { + return !dedicated[c] && uint32(c) >= p.numReservedForEVE + } + + var housekeeping, dedicatedCPUs, isolatedCPUs []cputopology.LCPU + for _, c := range p.allLCPUsSorted() { + if dedicated[c] { + dedicatedCPUs = append(dedicatedCPUs, c) + } else { + housekeeping = append(housekeeping, c) + } + } + known := map[cputopology.LCPU]bool{} + for _, c := range isolated { + if _, ok := p.topo.ByLCPU[c]; ok && !known[c] { + known[c] = true + isolatedCPUs = append(isolatedCPUs, c) + } + } + sort.Slice(isolatedCPUs, func(i, j int) bool { return isolatedCPUs[i] < isolatedCPUs[j] }) + + return []PoolUtilization{ + p.poolUtilization(PoolHousekeeping, housekeeping, isFree), + p.poolUtilization(PoolDedicated, dedicatedCPUs, isFree), + p.poolUtilization(PoolIsolated, isolatedCPUs, isFree), + } +} + +// poolUtilization summarises one pool. Caller must hold p.mu. +func (p *Placer) poolUtilization(pool CPUPool, cpus []cputopology.LCPU, + isFree func(cputopology.LCPU) bool) PoolUtilization { + + inPool := make(map[cputopology.LCPU]bool, len(cpus)) + for _, c := range cpus { + inPool[c] = true + } + out := PoolUtilization{ + Pool: pool, + CPUs: cpus, + TotalThreads: uint32(len(cpus)), + } + for _, c := range cpus { + if isFree(c) { + out.FreeCPUs = append(out.FreeCPUs, c) + } + } + out.FreeThreads = uint32(len(out.FreeCPUs)) + out.AllocatedThreads = out.TotalThreads - out.FreeThreads + + for i := range p.topo.Cores { + pc := &p.topo.Cores[i] + whole, allFree := true, true + for _, s := range pc.Siblings { + if !inPool[s] { + whole = false + break + } + if !isFree(s) { + allFree = false + } + } + if !whole { + continue + } + out.TotalCores++ + if allFree { + out.FreeWholeCores++ + } + } + return out +} diff --git a/pkg/pillar/cpuallocator/pools_test.go b/pkg/pillar/cpuallocator/pools_test.go new file mode 100644 index 00000000000..b23c9fc2e79 --- /dev/null +++ b/pkg/pillar/cpuallocator/pools_test.go @@ -0,0 +1,185 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cpuallocator + +import ( + "reflect" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/cputopology" + uuid "github.com/satori/go.uuid" +) + +// smt2Topology builds n physical cores with two SMT siblings each, numbered so +// that core i owns logical CPUs 2i and 2i+1 -- the layout the design doc's +// worked example uses. +func smt2Topology(n uint) *cputopology.Topology { + var infos []cputopology.CoreInfo + for core := uint(0); core < n; core++ { + for thread := uint(0); thread < 2; thread++ { + infos = append(infos, cputopology.CoreInfo{ + LCore: core*2 + thread, CoreID: core, + }) + } + } + return cputopology.BuildTopology(infos) +} + +func lcpus(ids ...uint32) []cputopology.LCPU { + out := make([]cputopology.LCPU, 0, len(ids)) + for _, id := range ids { + out = append(out, cputopology.LCPU(id)) + } + return out +} + +func mustReserve(t *testing.T, placer *Placer, id uuid.UUID, cpus ...uint32) { + t.Helper() + if err := placer.Reserve(id, cpus); err != nil { + t.Fatalf("Reserve(%s, %v): %v", id, cpus, err) + } +} + +func poolByKind(t *testing.T, pools []PoolUtilization, want CPUPool) PoolUtilization { + t.Helper() + for _, pool := range pools { + if pool.Pool == want { + return pool + } + } + t.Fatalf("no %s pool in %v", want, pools) + return PoolUtilization{} +} + +func checkPool(t *testing.T, got, want PoolUtilization) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("%s pool:\n got %+v\nwant %+v", want.Pool, got, want) + } +} + +// The worked example of the design doc (§9.3): an 8-thread node with CPU 0 +// reserved for EVE, app A holding one whole core and app B a single thread. +// Three of the free threads sit on cores that are already partly taken, so the +// node can still hand out exactly one whole core even though four threads are +// free -- which is the entire reason the whole-core counts are reported +// separately from the thread counts. +func TestPoolUtilization_WorkedExample(t *testing.T) { + placer := newPlacer(smt2Topology(4), 1) + mustReserve(t, placer, uuid.NewV5(uuid.NamespaceOID, "A"), 2, 3) + mustReserve(t, placer, uuid.NewV5(uuid.NamespaceOID, "B"), 4) + + pools := placer.PoolUtilization(lcpus(6, 7)) + if len(pools) != 3 { + t.Fatalf("want a report for every pool, got %d", len(pools)) + } + + checkPool(t, poolByKind(t, pools, PoolHousekeeping), PoolUtilization{ + Pool: PoolHousekeeping, + CPUs: lcpus(0, 1, 5, 6, 7), + FreeCPUs: lcpus(1, 5, 6, 7), + // cpu0 is reserved for EVE, so it counts as allocated. + TotalThreads: 5, AllocatedThreads: 1, FreeThreads: 4, + // core0 and core3 lie wholly in the pool; core1 and core2 straddle it. + TotalCores: 2, + // Only core3 is free whole: core0's sibling cpu0 belongs to EVE. + FreeWholeCores: 1, + }) + + checkPool(t, poolByKind(t, pools, PoolDedicated), PoolUtilization{ + Pool: PoolDedicated, + CPUs: lcpus(2, 3, 4), + TotalThreads: 3, AllocatedThreads: 3, FreeThreads: 0, + // Only core1 lies wholly in the pool; core2's cpu5 does not. + TotalCores: 1, FreeWholeCores: 0, + }) + + checkPool(t, poolByKind(t, pools, PoolIsolated), PoolUtilization{ + Pool: PoolIsolated, + CPUs: lcpus(6, 7), + FreeCPUs: lcpus(6, 7), + // The isolated set overlaps the housekeeping pool rather than + // partitioning with it: core3 is both free and kernel-isolated. + TotalThreads: 2, AllocatedThreads: 0, FreeThreads: 2, + TotalCores: 1, FreeWholeCores: 1, + }) +} + +// A free thread count cannot answer "does a whole-core workload fit?". With one +// thread taken on every core there are four free threads and no free core at +// all, so a consumer trusting the thread count would promise capacity that does +// not exist. +func TestPoolUtilization_FreeThreadsAreNotFreeCores(t *testing.T) { + placer := newPlacer(smt2Topology(4), 0) + for i, cpu := range []uint32{0, 2, 4, 6} { + mustReserve(t, placer, uuid.NewV5(uuid.NamespaceOID, string(rune('a'+i))), cpu) + } + housekeeping := poolByKind(t, placer.PoolUtilization(nil), PoolHousekeeping) + if housekeeping.FreeThreads != 4 { + t.Errorf("want 4 free threads, got %d", housekeeping.FreeThreads) + } + if housekeeping.FreeWholeCores != 0 { + t.Errorf("every core has a taken sibling, so none is free whole; got %d", + housekeeping.FreeWholeCores) + } + if housekeeping.TotalCores != 0 { + t.Errorf("every core straddles both pools, so none belongs wholly to "+ + "housekeeping; got %d", housekeeping.TotalCores) + } +} + +// An idle node reports its whole capacity as free, which is what a pre-flight +// "will it fit?" reads. +func TestPoolUtilization_IdleNode(t *testing.T) { + pools := newPlacer(smt2Topology(4), 2).PoolUtilization(nil) + + housekeeping := poolByKind(t, pools, PoolHousekeeping) + if housekeeping.TotalThreads != 8 || housekeeping.FreeThreads != 6 { + t.Errorf("want 8 threads with 6 free, got %d/%d", + housekeeping.TotalThreads, housekeeping.FreeThreads) + } + // Reserving CPUs 0 and 1 withholds core0 whole: a core with a housekeeping + // sibling is not a core a workload owns exclusively. + if housekeeping.FreeWholeCores != 3 { + t.Errorf("want 3 free whole cores, got %d", housekeeping.FreeWholeCores) + } + dedicated := poolByKind(t, pools, PoolDedicated) + if dedicated.TotalThreads != 0 || dedicated.CPUs != nil { + t.Errorf("nothing is dedicated, got %+v", dedicated) + } +} + +// Only the reported free whole-core count, never the free thread count, should +// agree with what the allocator will actually hand out. +func TestPoolUtilization_MatchesWhatAllocateGrants(t *testing.T) { + placer := newPlacer(smt2Topology(4), 1) + free := poolByKind(t, placer.PoolUtilization(nil), PoolHousekeeping).FreeWholeCores + + res := placer.Allocate(Request{ + UUID: uuid.NewV5(uuid.NamespaceOID, "greedy"), + NumVCPUs: int(free) * 2, + Mode: ModeWholeCoreSMT, + NUMA: NUMABestEffort, + }) + if res.Status != Success { + t.Fatalf("the report promised %d whole cores but Allocate said %v: %s", + free, res.Status, res.Message) + } + after := poolByKind(t, placer.PoolUtilization(nil), PoolHousekeeping) + if after.FreeWholeCores != 0 { + t.Errorf("all promised cores were taken, want 0 left, got %d", + after.FreeWholeCores) + } +} + +// Ids the topology does not know about must not appear in the report: an +// isolcpus list naming a CPU this kernel does not expose would otherwise be +// echoed back to the controller as capacity. +func TestPoolUtilization_UnknownIsolatedCPUsIgnored(t *testing.T) { + placer := newPlacer(smt2Topology(2), 0) + isolated := poolByKind(t, placer.PoolUtilization(lcpus(3, 99, 3)), PoolIsolated) + if !reflect.DeepEqual(isolated.CPUs, lcpus(3)) { + t.Errorf("want only the known, de-duplicated CPU 3, got %v", isolated.CPUs) + } +} From cf55ae8eb2fed46f52995a862bfff8be1a54dcde Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:00:27 +0000 Subject: [PATCH 03/15] types: add the CPU placement vocabulary and structured error codes Introduces the device-internal representation of a workload's CPU placement intent, mirroring the Kubernetes CPUManager and Topology Manager terms the controller API uses: cpu policy, full-pcpus-only, threads per core, NUMA policy, IO placement, isolation tier and disruption policy. Intent is kept deliberately separate from the allocator's vocabulary. Intent says what a workload needs; the allocator decides which host CPUs it gets. Keeping them apart means the wire format never dictates the placement mechanism, and it lets the two sources of intent -- the controller and the operator-editable /persist override -- resolve into one representation. The zero value means no policy was sent, so VmConfig.CPUsPinned alone keeps deciding and behaviour is unchanged for a controller that sets none of this. DomainStatus gains the resulting guest topology, the per-vCPU host CPU mapping, the emulator CPU set and the achieved placement quality. Quality is status rather than an error: a sub-optimally placed workload runs normally, and whether the improvement is worth a restart is a judgement for an operator. Adds the error-code registry reported alongside the free-text description, so a controller can distinguish conditions that need different responses -- a shortage a repack would fix, one nothing would fix, and a request that can never be satisfied -- without pattern-matching prose. ErrorDescription carries the code and a retry condition through to the wire. Signed-off-by: Mikhail Malyshev --- pkg/pillar/types/cpu_topology_test.go | 25 ++ pkg/pillar/types/cpuplacement.go | 380 ++++++++++++++++++++++++++ pkg/pillar/types/cpuplacement_test.go | 144 ++++++++++ pkg/pillar/types/domainmgrtypes.go | 46 +++- pkg/pillar/types/errorcodes.go | 42 +++ pkg/pillar/types/errortime.go | 12 +- pkg/pillar/types/zedmanagertypes.go | 7 + 7 files changed, 651 insertions(+), 5 deletions(-) create mode 100644 pkg/pillar/types/cpu_topology_test.go create mode 100644 pkg/pillar/types/cpuplacement.go create mode 100644 pkg/pillar/types/cpuplacement_test.go create mode 100644 pkg/pillar/types/errorcodes.go diff --git a/pkg/pillar/types/cpu_topology_test.go b/pkg/pillar/types/cpu_topology_test.go new file mode 100644 index 00000000000..12df4fc8303 --- /dev/null +++ b/pkg/pillar/types/cpu_topology_test.go @@ -0,0 +1,25 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package types + +import "testing" + +func TestCPUTopology_IsSet(t *testing.T) { + cases := []struct { + name string + topo CPUTopology + want bool + }{ + {"zero", CPUTopology{}, false}, + {"full", CPUTopology{Sockets: 1, Cores: 2, Threads: 2}, true}, + {"missing threads", CPUTopology{Sockets: 1, Cores: 2}, false}, + {"missing cores", CPUTopology{Sockets: 1, Threads: 2}, false}, + {"missing sockets", CPUTopology{Cores: 2, Threads: 2}, false}, + } + for _, c := range cases { + if got := c.topo.IsSet(); got != c.want { + t.Errorf("%s: IsSet()=%v want %v", c.name, got, c.want) + } + } +} diff --git a/pkg/pillar/types/cpuplacement.go b/pkg/pillar/types/cpuplacement.go new file mode 100644 index 00000000000..f688e22e620 --- /dev/null +++ b/pkg/pillar/types/cpuplacement.go @@ -0,0 +1,380 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" + + uuid "github.com/satori/go.uuid" +) + +// This file holds the per-app CPU placement *intent*: what the workload needs, +// expressed in the Kubernetes CPUManager / Topology Manager vocabulary that the +// controller API uses. It deliberately says nothing about which host CPUs are +// chosen -- that is the allocator's job (pkg/pillar/cpuallocator), and the two +// vocabularies are kept apart so the wire format never dictates the mechanism. +// +// The intent reaches domainmgr from two independent sources -- the controller +// (config.VmConfig) and the operator-editable /persist override -- which both +// resolve into the single representation below. + +// CPUPolicy expresses whether a workload gets host CPUs of its own. +type CPUPolicy uint8 + +const ( + // CPUPolicyUnspecified means the controller sent no policy; the legacy + // pin_cpu flag decides, preserving pre-policy behavior. + CPUPolicyUnspecified CPUPolicy = iota + // CPUPolicyShared is best-effort placement in the shared pool. + CPUPolicyShared + // CPUPolicyDedicated gives the workload host CPUs no other workload runs on. + CPUPolicyDedicated +) + +// String implements fmt.Stringer. +func (p CPUPolicy) String() string { + switch p { + case CPUPolicyUnspecified: + return "unspecified" + case CPUPolicyShared: + return "shared" + case CPUPolicyDedicated: + return "dedicated" + } + return fmt.Sprintf("unknown(%d)", uint8(p)) +} + +// CPUNUMAPolicy expresses how strictly a dedicated workload's CPUs must be +// confined to a single NUMA node. +type CPUNUMAPolicy uint8 + +const ( + // CPUNUMAPolicyUnspecified defaults to best-effort. + CPUNUMAPolicyUnspecified CPUNUMAPolicy = iota + // CPUNUMAPolicyNone expresses no NUMA preference. + CPUNUMAPolicyNone + // CPUNUMAPolicyBestEffort prefers one node but spans if it does not fit. + CPUNUMAPolicyBestEffort + // CPUNUMAPolicyRestricted behaves exactly like CPUNUMAPolicySingleNode + // today: the workload gets one NUMA node or it does not start. + // + // Kubernetes distinguishes the two by scope -- "restricted" minimises the + // nodes spanned across every resource aligned for the workload, while + // "single-numa-node" demands exactly one. That distinction needs a second + // resource to align against, and CPU placement currently considers only + // CPUs, so there is nothing for "minimise" to trade off and the two + // collapse. The value is kept in the vocabulary rather than rejected + // because the distinction becomes real once placement accounts for a + // workload's other assigned resources -- most immediately a passthrough + // PCI device, whose cores should come from the device's own NUMA node. + // That needs the per-device NUMA affinity the inventory can carry but does + // not yet populate. + CPUNUMAPolicyRestricted + // CPUNUMAPolicySingleNode requires one node; the workload fails to start + // rather than span. + CPUNUMAPolicySingleNode +) + +// String implements fmt.Stringer. +func (p CPUNUMAPolicy) String() string { + switch p { + case CPUNUMAPolicyUnspecified: + return "unspecified" + case CPUNUMAPolicyNone: + return "none" + case CPUNUMAPolicyBestEffort: + return "best-effort" + case CPUNUMAPolicyRestricted: + return "restricted" + case CPUNUMAPolicySingleNode: + return "single-numa-node" + } + return fmt.Sprintf("unknown(%d)", uint8(p)) +} + +// CPUIOPlacement selects where the hypervisor's non-vCPU threads run. +type CPUIOPlacement uint8 + +const ( + // CPUIOPlacementUnspecified defaults to dedicated. + CPUIOPlacementUnspecified CPUIOPlacement = iota + // CPUIOPlacementDedicated keeps emulator/IO threads in the workload's own + // CPU set. + CPUIOPlacementDedicated + // CPUIOPlacementHousekeeping pins emulator/IO threads off the hot cores so + // device emulation cannot steal cycles from busy vCPUs. + CPUIOPlacementHousekeeping +) + +// String implements fmt.Stringer. +func (p CPUIOPlacement) String() string { + switch p { + case CPUIOPlacementUnspecified: + return "unspecified" + case CPUIOPlacementDedicated: + return "dedicated" + case CPUIOPlacementHousekeeping: + return "housekeeping" + } + return fmt.Sprintf("unknown(%d)", uint8(p)) +} + +// CPUIsolationTier expresses how strongly the workload's CPUs must be shielded +// from interference. +type CPUIsolationTier uint8 + +const ( + // CPUIsolationTierUnspecified defaults to soft for dedicated workloads. + CPUIsolationTierUnspecified CPUIsolationTier = iota + // CPUIsolationTierNone is best-effort shared scheduling. + CPUIsolationTierNone + // CPUIsolationTierSoft is cpuset pinning plus topology-aware placement, + // applied at runtime. + CPUIsolationTierSoft + // CPUIsolationTierHard additionally sheds kernel housekeeping from the + // cores, which requires a kernel command-line change and a reboot. + CPUIsolationTierHard +) + +// String implements fmt.Stringer. +func (t CPUIsolationTier) String() string { + switch t { + case CPUIsolationTierUnspecified: + return "unspecified" + case CPUIsolationTierNone: + return "none" + case CPUIsolationTierSoft: + return "soft" + case CPUIsolationTierHard: + return "hard" + } + return fmt.Sprintf("unknown(%d)", uint8(t)) +} + +// SupportedBySoftIsolation reports whether this tier can be satisfied without a +// kernel command-line change. A request that cannot must fail closed rather +// than be silently downgraded. +func (t CPUIsolationTier) SupportedBySoftIsolation() bool { + return t != CPUIsolationTierHard +} + +// CPUDisruptionPolicy guards a running workload against collateral node-level +// disruption. +type CPUDisruptionPolicy uint8 + +const ( + // CPUDisruptionPolicyUnspecified defaults to allow. + CPUDisruptionPolicyUnspecified CPUDisruptionPolicy = iota + // CPUDisruptionPolicyAllow lets node-level disruptive actions proceed. + CPUDisruptionPolicyAllow + // CPUDisruptionPolicyProtect defers a node-level disruptive action that + // would take this workload down until the controller acknowledges it. + CPUDisruptionPolicyProtect +) + +// String implements fmt.Stringer. +func (p CPUDisruptionPolicy) String() string { + switch p { + case CPUDisruptionPolicyUnspecified: + return "unspecified" + case CPUDisruptionPolicyAllow: + return "allow" + case CPUDisruptionPolicyProtect: + return "protect" + } + return fmt.Sprintf("unknown(%d)", uint8(p)) +} + +// CPUPlacementPolicy is one workload's complete CPU placement intent. The zero +// value means "no policy", i.e. legacy behavior driven by VmConfig.CPUsPinned. +type CPUPlacementPolicy struct { + Policy CPUPolicy + FullPCPUsOnly bool + // ThreadsPerCore is how many SMT siblings of each dedicated core become + // vCPUs. 0 means unset; see EffectiveThreadsPerCore. + ThreadsPerCore uint32 + NUMAPolicy CPUNUMAPolicy + IOPlacement CPUIOPlacement + IsolationTier CPUIsolationTier + DisruptionPolicy CPUDisruptionPolicy +} + +// IsDedicated reports whether the workload asked for CPUs of its own. +func (p CPUPlacementPolicy) IsDedicated() bool { + return p.Policy == CPUPolicyDedicated +} + +// IsTopologyAware reports whether the intent calls for whole-physical-core, +// SMT/NUMA-aware placement. Dedicated alone is not enough: without +// full-pcpus-only the workload is allocated at SMT-thread granularity and may +// share a physical core, which is the legacy pinning behavior. +func (p CPUPlacementPolicy) IsTopologyAware() bool { + return p.IsDedicated() && p.FullPCPUsOnly +} + +// EffectiveThreadsPerCore resolves the unset case to 2, i.e. both SMT siblings +// become vCPUs. The API documents that default as applying on SMT hardware; EVE +// applies it unconditionally, so on a node without SMT a request that leaves +// threads_per_core unset cannot be satisfied and must set threads_per_core=1. +func (p CPUPlacementPolicy) EffectiveThreadsPerCore() uint32 { + if p.ThreadsPerCore == 0 { + return 2 + } + return p.ThreadsPerCore +} + +// CPUPlacementQuality is how good a workload's actual CPU placement is, +// reported so an operator or controller can decide whether a disruptive repack +// is worth it. It is status, not an error: a sub-optimally placed workload runs +// normally, and nothing about it needs fixing unless someone judges the +// improvement worth a restart. +type CPUPlacementQuality uint8 + +const ( + // CPUPlacementQualityUnspecified means placement quality was not evaluated, + // which is the case for any workload that is not whole-core pinned. + CPUPlacementQualityUnspecified CPUPlacementQuality = iota + // CPUPlacementQualityOptimal means no better placement exists for this + // workload -- including the common case where the workload got different + // CPUs than first proposed but ones that are just as good. + CPUPlacementQualityOptimal + // CPUPlacementQualityNeedsRepack means a better placement exists but only + // by moving workloads that are already running. Moving a running workload + // means restarting it, so this is reported and left to the operator rather + // than acted on. + CPUPlacementQualityNeedsRepack +) + +// String implements fmt.Stringer. +func (q CPUPlacementQuality) String() string { + switch q { + case CPUPlacementQualityUnspecified: + return "unspecified" + case CPUPlacementQualityOptimal: + return "optimal" + case CPUPlacementQualityNeedsRepack: + return "needs-repack" + } + return fmt.Sprintf("unknown(%d)", uint8(q)) +} + +// CPUPoolKind names one partition of the node's logical CPUs in the node-level +// CPU pool report. Mirrors the eve-api info.CPUPoolKind enum. +type CPUPoolKind uint8 + +const ( + // CPUPoolKindUnspecified is the unset zero value. + CPUPoolKindUnspecified CPUPoolKind = iota + // CPUPoolKindHousekeeping is EVE's own CPUs plus every CPU no workload + // holds exclusively, which is where non-dedicated workloads run. + CPUPoolKindHousekeeping + // CPUPoolKindDedicated is the CPUs handed out exclusively to workloads with + // a dedicated CPU policy, including siblings parked idle by a + // one-thread-per-core request. + CPUPoolKindDedicated + // CPUPoolKindIsolated is the set the running kernel isolates (isolcpus). + // It overlaps the other two rather than partitioning with them. + CPUPoolKindIsolated +) + +// String implements fmt.Stringer. +func (k CPUPoolKind) String() string { + switch k { + case CPUPoolKindUnspecified: + return "unspecified" + case CPUPoolKindHousekeeping: + return "housekeeping" + case CPUPoolKindDedicated: + return "dedicated" + case CPUPoolKindIsolated: + return "isolated" + } + return fmt.Sprintf("unknown(%d)", uint8(k)) +} + +// CPUPoolUtilization is one CPU pool's extent and how much of it is still +// available. The whole-core counts are reported alongside the thread counts +// because they are not derivable from each other: a free thread on a +// partially-taken core cannot satisfy a request for whole physical cores, so a +// single "free" number answers one of the two request shapes wrongly. +type CPUPoolUtilization struct { + Kind CPUPoolKind + // CPUs is every logical CPU in the pool, ascending. + CPUs []uint32 + // FreeCPUs is the subset of CPUs that could still be given to a workload + // asking for dedicated placement. + FreeCPUs []uint32 + TotalThreads uint32 + AllocatedThreads uint32 + FreeThreads uint32 + // TotalCores counts physical cores all of whose SMT siblings are in CPUs. + TotalCores uint32 + // FreeWholeCores counts physical cores all of whose SMT siblings are free -- + // the number that bounds how many more whole-core workloads fit. + FreeWholeCores uint32 +} + +// AppCPUDemand is one application's CPU-relevant configuration, as the +// controller expressed it. It carries only what a placement decision needs -- +// not the whole VmConfig -- so the demand set stays independent of everything +// else an app config describes. +// +// The intent here is the controller's alone. The operator-editable /persist +// override is read by domainmgr, which zedmanager knows nothing about, so an +// entry saying "not pinned" does not mean the workload will not be pinned. +type AppCPUDemand struct { + UUID uuid.UUID + // DisplayName is for logs and diagnostics only; nothing keys off it. + DisplayName string + VCpus int + // CPUsPinned is the legacy pin flag, which still decides when the + // controller sent no CPUPlacement policy. + CPUsPinned bool + CPUPlacement CPUPlacementPolicy +} + +// CPUDemandSet is every application intended to run on this node, with its CPU +// intent -- the demand domainmgr plans CPU placement against. +// +// It exists because a DomainConfig is not published until an app's volumes and +// network are ready, so domainmgr's view of "the configured apps" is really +// "the apps that finished downloading first". Planning against that makes the +// layout depend on image download order: whichever app activates first is +// planned as if it were alone and takes CPUs the full plan would have given to +// another. zedmanager knows the whole intended set from the moment the config +// arrives, so it publishes it here. +// +// It is deliberately one aggregate object rather than one item per app: a +// per-app topic would only move the same "what has arrived so far" problem onto +// a faster topic. Receiving one object that IS the whole set makes +// set-completeness atomic -- the subscriber replaces its view wholesale. +// +// An empty set is published explicitly, so "no pinned apps are configured" is +// distinguishable from "zedmanager has not spoken yet". +type CPUDemandSet struct { + // Apps is sorted by UUID, so an unchanged set is byte-identical between + // publications and does not look like a change. + Apps []AppCPUDemand +} + +// Key returns the pubsub key. There is one demand set per node. +func (s CPUDemandSet) Key() string { + return "global" +} + +// CPUPoolStatus is the node's CPU pool report: how the logical CPUs are +// partitioned and how much of each partition is left. +// +// domainmgr owns the allocator, so it is the only agent that can compute this; +// zedagent subscribes and projects it onto ZInfoDevice.cpu_pools so a controller +// can answer "will this workload fit?" before a deploy and explain a placement +// failure after one. Republished whenever the dedicated set changes. +type CPUPoolStatus struct { + Pools []CPUPoolUtilization +} + +// Key returns the pubsub key. There is one report per node. +func (s CPUPoolStatus) Key() string { + return "global" +} diff --git a/pkg/pillar/types/cpuplacement_test.go b/pkg/pillar/types/cpuplacement_test.go new file mode 100644 index 00000000000..d939582c69c --- /dev/null +++ b/pkg/pillar/types/cpuplacement_test.go @@ -0,0 +1,144 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package types + +import "testing" + +func TestCPUPlacementIsTopologyAware(t *testing.T) { + tests := []struct { + name string + p CPUPlacementPolicy + want bool + }{ + {"zero value is legacy", CPUPlacementPolicy{}, false}, + {"shared is not topology-aware", CPUPlacementPolicy{Policy: CPUPolicyShared}, false}, + { + "dedicated without full-pcpus-only stays thread-granular", + CPUPlacementPolicy{Policy: CPUPolicyDedicated}, + false, + }, + { + "dedicated with full-pcpus-only", + CPUPlacementPolicy{Policy: CPUPolicyDedicated, FullPCPUsOnly: true}, + true, + }, + { + "full-pcpus-only alone means nothing without a dedicated policy", + CPUPlacementPolicy{FullPCPUsOnly: true}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.p.IsTopologyAware(); got != tt.want { + t.Errorf("IsTopologyAware() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCPUPlacementIsDedicated(t *testing.T) { + if (CPUPlacementPolicy{}).IsDedicated() { + t.Error("zero value must not be dedicated") + } + if (CPUPlacementPolicy{Policy: CPUPolicyShared}).IsDedicated() { + t.Error("shared must not be dedicated") + } + if !(CPUPlacementPolicy{Policy: CPUPolicyDedicated}).IsDedicated() { + t.Error("dedicated policy must report dedicated") + } +} + +// A dedicated workload that asks for whole cores and does not say how many +// threads per core gets both siblings, matching the API's documented default. +func TestCPUPlacementEffectiveThreadsPerCore(t *testing.T) { + tests := []struct { + name string + p CPUPlacementPolicy + want uint32 + }{ + {"unset defaults to 2", CPUPlacementPolicy{Policy: CPUPolicyDedicated, FullPCPUsOnly: true}, 2}, + {"explicit 1 is honored", CPUPlacementPolicy{Policy: CPUPolicyDedicated, FullPCPUsOnly: true, ThreadsPerCore: 1}, 1}, + {"explicit 2 is honored", CPUPlacementPolicy{Policy: CPUPolicyDedicated, FullPCPUsOnly: true, ThreadsPerCore: 2}, 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.p.EffectiveThreadsPerCore(); got != tt.want { + t.Errorf("EffectiveThreadsPerCore() = %d, want %d", got, tt.want) + } + }) + } +} + +// The isolation tier a workload may request is gated by what the device can +// actually do; only "none" and "soft" are implementable without a kernel +// command-line change. +func TestCPUIsolationTierSupported(t *testing.T) { + supported := []CPUIsolationTier{ + CPUIsolationTierUnspecified, + CPUIsolationTierNone, + CPUIsolationTierSoft, + } + for _, tier := range supported { + if !tier.SupportedBySoftIsolation() { + t.Errorf("tier %v should be satisfiable by soft isolation", tier) + } + } + if CPUIsolationTierHard.SupportedBySoftIsolation() { + t.Error("hard isolation needs a kernel cmdline change and must not be claimed as satisfied") + } +} + +// Strings appear in logs and status; they must be stable and not panic on values +// outside the known range. Every enumerator is listed, so an added value without +// a String() case is caught here rather than seen as "unknown(N)" in the field. +func TestCPUPlacementStrings(t *testing.T) { + cases := []struct { + name, got, want string + }{ + {"CPUPolicyUnspecified", CPUPolicyUnspecified.String(), "unspecified"}, + {"CPUPolicyShared", CPUPolicyShared.String(), "shared"}, + {"CPUPolicyDedicated", CPUPolicyDedicated.String(), "dedicated"}, + {"CPUPolicy fallback", CPUPolicy(200).String(), "unknown(200)"}, + + {"CPUNUMAPolicyUnspecified", CPUNUMAPolicyUnspecified.String(), "unspecified"}, + {"CPUNUMAPolicyNone", CPUNUMAPolicyNone.String(), "none"}, + {"CPUNUMAPolicyBestEffort", CPUNUMAPolicyBestEffort.String(), "best-effort"}, + {"CPUNUMAPolicyRestricted", CPUNUMAPolicyRestricted.String(), "restricted"}, + {"CPUNUMAPolicySingleNode", CPUNUMAPolicySingleNode.String(), "single-numa-node"}, + {"CPUNUMAPolicy fallback", CPUNUMAPolicy(200).String(), "unknown(200)"}, + + {"CPUIOPlacementUnspecified", CPUIOPlacementUnspecified.String(), "unspecified"}, + {"CPUIOPlacementDedicated", CPUIOPlacementDedicated.String(), "dedicated"}, + {"CPUIOPlacementHousekeeping", CPUIOPlacementHousekeeping.String(), "housekeeping"}, + {"CPUIOPlacement fallback", CPUIOPlacement(200).String(), "unknown(200)"}, + + {"CPUIsolationTierUnspecified", CPUIsolationTierUnspecified.String(), "unspecified"}, + {"CPUIsolationTierNone", CPUIsolationTierNone.String(), "none"}, + {"CPUIsolationTierSoft", CPUIsolationTierSoft.String(), "soft"}, + {"CPUIsolationTierHard", CPUIsolationTierHard.String(), "hard"}, + {"CPUIsolationTier fallback", CPUIsolationTier(200).String(), "unknown(200)"}, + + {"CPUDisruptionPolicyUnspecified", CPUDisruptionPolicyUnspecified.String(), "unspecified"}, + {"CPUDisruptionPolicyAllow", CPUDisruptionPolicyAllow.String(), "allow"}, + {"CPUDisruptionPolicyProtect", CPUDisruptionPolicyProtect.String(), "protect"}, + {"CPUDisruptionPolicy fallback", CPUDisruptionPolicy(200).String(), "unknown(200)"}, + + {"CPUPlacementQualityUnspecified", CPUPlacementQualityUnspecified.String(), "unspecified"}, + {"CPUPlacementQualityOptimal", CPUPlacementQualityOptimal.String(), "optimal"}, + {"CPUPlacementQualityNeedsRepack", CPUPlacementQualityNeedsRepack.String(), "needs-repack"}, + {"CPUPlacementQuality fallback", CPUPlacementQuality(200).String(), "unknown(200)"}, + + {"CPUPoolKindUnspecified", CPUPoolKindUnspecified.String(), "unspecified"}, + {"CPUPoolKindHousekeeping", CPUPoolKindHousekeeping.String(), "housekeeping"}, + {"CPUPoolKindDedicated", CPUPoolKindDedicated.String(), "dedicated"}, + {"CPUPoolKindIsolated", CPUPoolKindIsolated.String(), "isolated"}, + {"CPUPoolKind fallback", CPUPoolKind(200).String(), "unknown(200)"}, + } + for _, c := range cases { + if c.got != c.want { + t.Errorf("%s: String() = %q, want %q", c.name, c.got, c.want) + } + } +} diff --git a/pkg/pillar/types/domainmgrtypes.go b/pkg/pillar/types/domainmgrtypes.go index ff5054148ba..d52274a1e43 100644 --- a/pkg/pillar/types/domainmgrtypes.go +++ b/pkg/pillar/types/domainmgrtypes.go @@ -285,8 +285,14 @@ type VmConfig struct { VncDisplay uint32 VncPasswd string CPUsPinned bool - VMMMaxMem int // in kbytes - EnableVncShimVM bool + // CPUPlacement is the controller's CPU placement intent (see + // cpuplacement.go). Its zero value means the controller sent no policy, in + // which case CPUsPinned alone decides, exactly as before this field existed. + // When a policy is present it is authoritative and CPUsPinned is derived + // from it. + CPUPlacement CPUPlacementPolicy + VMMMaxMem int // in kbytes + EnableVncShimVM bool // Enables enforcement of user-defined ordering for network interfaces. EnforceNetworkInterfaceOrder bool // EnableOemWinLicenseKey indicates the app should receive the embedded Windows license key (if available) @@ -302,6 +308,17 @@ type VmConfig struct { BootOrder zcommon.BootOrder } +// CPUTopology is the guest-visible CPU topology emitted to QEMU -smp. +type CPUTopology struct { + Sockets int + Cores int + Threads int +} + +// IsSet reports whether a non-legacy (computed) topology is present. +// Zero value means "fall back to the legacy flat VCpus topology". +func (t CPUTopology) IsSet() bool { return t.Sockets > 0 && t.Cores > 0 && t.Threads > 0 } + // VmMode is the type for the virtualization mode type VmMode uint8 @@ -461,6 +478,22 @@ type DomainStatus struct { HoldUntil time.Time // GdbSocket is the gdbstub UNIX socket exposed for a held domain, if any. GdbSocket string + + // VMTopology is the guest CPU topology to expose via QEMU -smp. + // Zero value (see CPUTopology.IsSet) means legacy flat VCpus topology. + VMTopology CPUTopology + // OrderedCPUs maps guest vCPU index -> host logical CPU for strict 1:1 + // pinning (populated only for topology-pinned VMs). + OrderedCPUs []uint32 + // EmulatorCPUs is the housekeeping CPU set that QEMU emulator/IO threads + // are pinned to. Populated only for a topology-pinned VM that asked for + // io_placement=housekeeping, and only when a CPU free of every pinned + // workload was available; empty means the emulator threads stay on the VM's + // own dedicated cores. + EmulatorCPUs []uint32 + // PlacementQuality reports how good the achieved CPU placement is, so a + // sub-optimal placement can be surfaced without failing the workload. + PlacementQuality CPUPlacementQuality } func (status DomainStatus) Key() string { @@ -723,7 +756,14 @@ type Capabilities struct { HWAssistedVirtualization bool // VMX/SVM for amd64 or Arm virtualization extensions for arm64 IOVirtualization bool // I/O Virtualization support CPUPinning bool // CPU Pinning support - UseVHost bool // vHost support + // CPUTopologyPinning is whether the hypervisor can bind each guest vCPU to + // one named host CPU and expose the resulting SMT topology to the guest. + // Strictly stronger than CPUPinning, which only confines a domain to a + // cpuset: whole-physical-core placement is meaningless without it, since + // the guest neither learns which of its vCPUs are SMT siblings nor stays + // on the cores it was placed on. + CPUTopologyPinning bool + UseVHost bool // vHost support } // WatchdogParam is used in some proc functions that have a timeout, diff --git a/pkg/pillar/types/errorcodes.go b/pkg/pillar/types/errorcodes.go new file mode 100644 index 00000000000..f43385b5d32 --- /dev/null +++ b/pkg/pillar/types/errorcodes.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package types + +// Machine-parseable error codes reported alongside the free-text description in +// ErrorInfo.error_code, so a controller can react programmatically without +// parsing prose. Codes are namespaced by domain and are part of the published +// contract: once shipped, a code's meaning must not change. New conditions get +// new codes rather than redefining existing ones. +const ( + // ErrorCodeCPUPlacementInsufficient means the node does not have enough + // CPUs of the required kind for this workload, in any arrangement. + ErrorCodeCPUPlacementInsufficient = "cpu.placement.insufficient" + // ErrorCodeCPUPlacementNeedsRepack means a placement exists, but only by + // moving workloads that are already running. + ErrorCodeCPUPlacementNeedsRepack = "cpu.placement.needs_repack" + // ErrorCodeCPUPlacementDegraded would mean the workload is placed and + // running, but not at the best achievable quality. + // + // Reserved: nothing emits it. The running-but-improvable case is reported as + // ErrorCodeCPUPlacementNeedsRepack, which names the reason a better + // placement is not taken (it would restart running workloads). The code is + // kept because "degraded" is where a fragmentation advisory about a resource + // other than cores would land, and a controller must not be given a second + // meaning for a code it already handles. + ErrorCodeCPUPlacementDegraded = "cpu.placement.degraded" + // ErrorCodeCPUPolicyOddVCPU means whole-core-SMT was requested with an odd + // vCPU count, which no arrangement of two-thread cores can satisfy. + ErrorCodeCPUPolicyOddVCPU = "cpu.policy.odd_vcpu" + // ErrorCodeCPUIsolationTierUnavailable means the requested isolation tier + // is not supported by this node. + ErrorCodeCPUIsolationTierUnavailable = "cpu.isolation.tier_unavailable" + // ErrorCodeCPUPolicyInvalid means the placement policy is malformed or + // self-contradictory. + ErrorCodeCPUPolicyInvalid = "cpu.policy.invalid" + // ErrorCodeCPUTopologyUnsupported means whole-physical-core placement was + // requested but the active hypervisor cannot bind vCPUs to named host CPUs + // or expose the resulting guest topology, so the request cannot be honored + // on this node no matter how many CPUs are free. + ErrorCodeCPUTopologyUnsupported = "cpu.topology.unsupported" +) diff --git a/pkg/pillar/types/errortime.go b/pkg/pillar/types/errortime.go index 48b22a6c0e9..22aee19d2b8 100644 --- a/pkg/pillar/types/errortime.go +++ b/pkg/pillar/types/errortime.go @@ -86,8 +86,13 @@ type ErrorEntity struct { // ErrorDescription contains error details type ErrorDescription struct { - Error string - ErrorTime time.Time + Error string + ErrorTime time.Time + // ErrorCode is a machine-parseable, namespaced token identifying the + // condition (see errorcodes.go), reported alongside the free-text Error so + // a controller can react programmatically instead of matching prose. + // Empty when the producer has no specific code for the failure. + ErrorCode string ErrorSeverity ErrorSeverity ErrorRetryCondition string ErrorEntities []*ErrorEntity @@ -117,6 +122,7 @@ func (ed *ErrorDescription) ToProto() *info.ErrorInfo { } errInfo := new(info.ErrorInfo) errInfo.Description = ed.Error + errInfo.ErrorCode = ed.ErrorCode errInfo.Timestamp = timestamppb.New(ed.ErrorTime) errInfo.Severity = info.Severity(ed.ErrorSeverity) errInfo.RetryCondition = ed.ErrorRetryCondition @@ -154,6 +160,7 @@ func (etPtr *ErrorAndTime) SetError(errStr string, errorTime time.Time) { // ClearError removes it func (etPtr *ErrorAndTime) ClearError() { etPtr.Error = "" + etPtr.ErrorCode = "" etPtr.ErrorTime = time.Time{} etPtr.ErrorRetryCondition = "" etPtr.ErrorSeverity = ErrorSeverityUnspecified @@ -237,6 +244,7 @@ func (etsPtr *ErrorAndTimeWithSource) IsErrorSource(source interface{}) bool { // ClearErrorWithSource - Clears error state func (etsPtr *ErrorAndTimeWithSource) ClearErrorWithSource() { etsPtr.Error = "" + etsPtr.ErrorCode = "" etsPtr.ErrorSourceType = "" etsPtr.ErrorTime = time.Time{} etsPtr.ErrorRetryCondition = "" diff --git a/pkg/pillar/types/zedmanagertypes.go b/pkg/pillar/types/zedmanagertypes.go index 56cba72ab0e..5312802da74 100644 --- a/pkg/pillar/types/zedmanagertypes.go +++ b/pkg/pillar/types/zedmanagertypes.go @@ -350,6 +350,13 @@ type AppInstanceStatus struct { NoUploadStatsToController bool // Am I Cluster Designated Node Id for this app IsDesignatedNodeID bool + // PlacementQuality mirrors DomainStatus.PlacementQuality: how good the CPU + // placement the workload actually got is. It is advisory status, not an + // error -- a workload reported as needing a repack is running normally -- + // so it is kept out of ErrorAndTimeWithSource, where it would make every + // HasError() caller treat a sub-optimally placed app as a failed one. + // zedagent turns it into a WARNING-severity advisory on the app's info. + PlacementQuality CPUPlacementQuality } // AppCount is uint8 and it should be sufficient for the number of apps we can support From 1f5910f23ca0ee02c8ccfbe6e190c29815ca5ca6 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:00:47 +0000 Subject: [PATCH 04/15] hypervisor: expose the guest CPU topology and pin vCPUs 1:1 Realizing a whole-core placement on QEMU/KVM has three parts. The guest is launched with an -smp topology computed from the assignment, so software inside it sees the real SMT structure and can place its own hot work on non-sibling cores. A poll-mode datapath deliberately runs a worker on each sibling; without a truthful topology it cannot tell which vCPUs share a core. Each vCPU thread is then pinned 1:1 to its assigned host CPU. QEMU is already started paused, so the vCPU threads exist while the guest has not executed and there is no pre-pin race. The guest-vCPU-to-host-thread mapping comes from QMP query-cpus-fast, which is the only place it exists: QEMU does not name its vCPU threads unless started with debug-threads=on, and a domain's thread group also holds vhost_task helpers that modern kernels create as user threads in that same group, indistinguishable from vCPU threads by name or by flags. The pin is applied after the cgroup cpuset has been written and before the guest is released, so it is not undone by the cpuset. Under io_placement=housekeeping the non-vCPU threads are pinned off the hot cores, so device emulation cannot steal cycles from a busy vCPU. A virtio-blk iothread keeps disk IO off the main loop. Kubevirt reports that it cannot bind individual vCPUs. The capability is separate from plain cpuset confinement, because a hypervisor that can confine a domain to a set of CPUs may still be unable to bind one vCPU to one CPU or to advertise the resulting topology -- and accepting a whole-core request it cannot apply would report the workload as optimally placed while nothing was pinned. Signed-off-by: Mikhail Malyshev --- pkg/pillar/hypervisor/kubevirt.go | 7 +- pkg/pillar/hypervisor/kvm.go | 108 +++++++++++++++++---- pkg/pillar/hypervisor/kvm_test.go | 152 ++++++++++++++++++++++++++++++ pkg/pillar/hypervisor/pinning.go | 133 ++++++++++++++++++++++++++ pkg/pillar/hypervisor/qmp.go | 45 +++++++++ pkg/pillar/hypervisor/qmp_test.go | 68 +++++++++++++ 6 files changed, 494 insertions(+), 19 deletions(-) create mode 100644 pkg/pillar/hypervisor/pinning.go diff --git a/pkg/pillar/hypervisor/kubevirt.go b/pkg/pillar/hypervisor/kubevirt.go index 4cf525dfed1..bae33f3de35 100644 --- a/pkg/pillar/hypervisor/kubevirt.go +++ b/pkg/pillar/hypervisor/kubevirt.go @@ -252,7 +252,12 @@ func (ctx kubevirtContext) GetCapabilities() (*types.Capabilities, error) { HWAssistedVirtualization: true, IOVirtualization: vtd, CPUPinning: true, - UseVHost: false, // kubevirt does not support vhost yet + // KubeVirt's dedicatedCpuPlacement confines the VMI to exclusive CPUs, + // but EVE cannot name which host CPU each vCPU lands on nor emit a + // guest SMT topology through it, so whole-core placement cannot be + // applied here and must be refused rather than reported as done. + CPUTopologyPinning: false, + UseVHost: false, // kubevirt does not support vhost yet } return ctx.capabilities, nil } diff --git a/pkg/pillar/hypervisor/kvm.go b/pkg/pillar/hypervisor/kvm.go index 2730afc3846..d53d248f101 100644 --- a/pkg/pillar/hypervisor/kvm.go +++ b/pkg/pillar/hypervisor/kvm.go @@ -257,9 +257,21 @@ const qemuGlobalConfTemplate = `# This file is automatically generated by domain [smp-opts] cpus = "{{.DomainConfig.VCpus}}" +{{- if .VMTopology.IsSet}} + sockets = "{{.VMTopology.Sockets}}" + cores = "{{.VMTopology.Cores}}" + threads = "{{.VMTopology.Threads}}" +{{- else}} sockets = "1" cores = "{{.DomainConfig.VCpus}}" threads = "1" +{{- end}} + +{{- if .EmulatorCPUs}} + +[object "iothread0"] + qom-type = "iothread" +{{- end}} [device] driver = "virtio-serial" @@ -437,6 +449,9 @@ const qemuDiskTemplate = ` driver = "virtio-blk-pci" bus = "pci.{{.PCIId}}" addr = "0x0" +{{- if .UseIOThread}} + iothread = "iothread0" +{{- end}} {{- end}} drive = "drive-virtio-disk{{.DiskID}}" {{- else}} @@ -642,6 +657,10 @@ type tQemuDiskContext struct { Machine string PCIId, DiskID, SATAId, NumQueues int AioType string + // UseIOThread attaches virtio-blk to the iothread0 object declared by the + // global template. Both are emitted together or not at all; see + // wantsBlockIOThread for when and why. + UseIOThread bool types.DiskStatus } @@ -729,6 +748,9 @@ type KvmContext struct { dmCPUArgs []string dmFmlCPUArgs []string capabilities *types.Capabilities + // status is bound per-domain by Task(); it carries OrderedCPUs/EmulatorCPUs + // so Start() can apply topology pinning without any ephemeral hand-off state. + status *types.DomainStatus } func newKvm() Hypervisor { @@ -777,7 +799,10 @@ func (ctx KvmContext) GetCapabilities() (*types.Capabilities, error) { HWAssistedVirtualization: true, IOVirtualization: vtd, CPUPinning: true, - UseVHost: true, + // pinDomainThreads binds each vCPU thread over QMP and the domain + // config emits the matching -smp sockets/cores/threads. + CPUTopologyPinning: true, + UseVHost: true, } return ctx.capabilities, nil } @@ -814,6 +839,9 @@ func (ctx KvmContext) Task(status *types.DomainStatus) types.Task { if status.VirtualizationMode == types.NOHYPER { return ctx.ctrdContext } + // Bind the domain's status so Start() can read OrderedCPUs/EmulatorCPUs for + // topology pinning (value receiver: this sets the field on the returned copy). + ctx.status = status return ctx } @@ -878,6 +906,34 @@ func cleanupOVMFSettings(domainName string) error { return nil } +// legacyThreadContextArgs returns the qemu arguments for the legacy CPU pinning +// path, or nil when that path does not apply. CPU affinity cannot be expressed +// in the .ini file, hence the command line. The format is +// -object thread-context,id=tc1,cpu-affinity=0-1,cpu-affinity=6-7 and the +// thread-context object exists since qemu 7.2. +// +// Legacy pinning and topology pinning are two mechanisms for the same job and +// only one may be in force, which is what OrderedCPUs selects between. +// thread-context carries a single affinity mask, so it can only confine the +// domain's threads to status.CPUs as a set; it cannot say which host CPU a given +// guest vCPU gets, nor hold emulator threads and vCPU threads apart. Both of +// those are the whole point of the topology path, which instead pins each thread +// individually over QMP in Start() (see pinning.go). Emitting the mask as well +// would state a placement that contradicts the intended one: for a topology- +// pinned domain status.CPUs is the union of the vCPU cores, the SMT siblings +// parked idle next to them and -- under io_placement=housekeeping -- the shared +// housekeeping pool. +func legacyThreadContextArgs(config types.DomainConfig, status types.DomainStatus) []string { + if !config.CPUsPinned || len(status.OrderedCPUs) > 0 { + return nil + } + threadContext := "thread-context,id=tc1" + for _, cpu := range status.CPUs { + threadContext += fmt.Sprintf(",cpu-affinity=%d", cpu) + } + return []string{"-object", threadContext} +} + // Setup sets up kvm // Note: globalConfig can be nil only in unit tests. In production, it is always // provided by domainmgr from agentlog.GetGlobalConfig(). @@ -960,19 +1016,7 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig } } - // Add CPUs affinity as a parameter to qemu. - // It's not supported to be configured in the .ini file so we need to add it here. - // The arguments are in the format of: -object thread-context,id=tc1,cpu-affinity=0-1,cpu-affinity=6-7 - // The thread-context object is introduced in qemu 7.2 - if config.CPUsPinned { - // Create the thread-context object string - threadContext := "thread-context,id=tc1" - for _, cpu := range status.CPUs { - // Add the cpu-affinity arguments to the thread-context object - threadContext += fmt.Sprintf(",cpu-affinity=%d", cpu) - } - args = append(args, "-object", threadContext) - } + args = append(args, legacyThreadContextArgs(config, status)...) spec, err := ctx.setupSpec(&status, &config, status.OCIConfigDir) @@ -1722,6 +1766,24 @@ func detectIntelIGPU(adapters []types.IoAdapter, aa *types.AssignableAdapters) b return false } +// wantsBlockIOThread reports whether this domain gets a dedicated iothread for +// its virtio-blk devices instead of running the block datapath on the QEMU main +// loop. +// +// Gated on EmulatorCPUs, i.e. on io_placement=housekeeping having actually been +// granted a housekeeping CPU set, because that is the only configuration in +// which the block datapath is meant to live somewhere other than where the +// vCPUs run: pinDomainThreads moves every non-vCPU thread onto that set, and +// giving virtio-blk a thread of its own is what keeps its work from being +// serialised behind the main loop's other duties there. Any other domain -- +// unpinned, legacy-pinned, or topology-pinned with io_placement=dedicated -- +// keeps the pre-existing single-threaded layout, since moving its block IO off +// the main loop would change the IO behaviour of every VM on every device +// without buying any vCPU isolation. +func wantsBlockIOThread(status types.DomainStatus) bool { + return len(status.EmulatorCPUs) > 0 +} + // CreateDomConfig creates a domain config (a qemu config file, // typically named something like xen-%d.cfg) func (ctx KvmContext) CreateDomConfig(domainName string, @@ -1813,10 +1875,11 @@ func (ctx KvmContext) CreateDomConfig(domainName string, // render disk device model settings diskContext := tQemuDiskContext{ - Machine: ctx.devicemodel, - PCIId: 4, - AioType: "io_uring", - NumQueues: config.VCpus, + Machine: ctx.devicemodel, + PCIId: 4, + AioType: "io_uring", + NumQueues: config.VCpus, + UseIOThread: wantsBlockIOThread(status), } for _, ds := range diskStatusList { if ds.Devtype == "" { @@ -2043,6 +2106,15 @@ func (ctx KvmContext) Start(domainName string) error { } } + var ordered, emulator []uint32 + if ctx.status != nil { + ordered = ctx.status.OrderedCPUs + emulator = ctx.status.EmulatorCPUs + } + if err := ctx.pinDomainThreads(domainName, qmpFile, ordered, emulator); err != nil { + return logError("failed to pin CPU threads for domain %s: %v", domainName, err) + } + if err := execContinue(qmpFile); err != nil { return logError("failed to start domain that is stopped %v", err) } diff --git a/pkg/pillar/hypervisor/kvm_test.go b/pkg/pillar/hypervisor/kvm_test.go index eeb35391417..29261e4ed1c 100644 --- a/pkg/pillar/hypervisor/kvm_test.go +++ b/pkg/pillar/hypervisor/kvm_test.go @@ -3422,3 +3422,155 @@ func TestDecideKvmState(t *testing.T) { }) } } + +// renderDomConfig writes a domain config for the given config/status pair and +// returns it, so a test can assert on one block instead of a whole golden file. +func renderDomConfig(t *testing.T, ctx KvmContext, config types.DomainConfig, + status types.DomainStatus, disks []types.DiskStatus) string { + t.Helper() + conf, err := os.CreateTemp("/tmp", "config") + if err != nil { + t.Fatalf("can't create config file for a domain: %v", err) + } + defer os.Remove(conf.Name()) + aa := types.AssignableAdapters{Initialized: true} + if err := ctx.CreateDomConfig(DefaultDomainName, config, status, disks, + &aa, nil, swtpmCtrlSock, conf); err != nil { + t.Fatalf("CreateDomConfig failed: %v", err) + } + out, err := os.ReadFile(conf.Name()) + if err != nil { + t.Fatalf("reading conf file failed: %v", err) + } + return string(out) +} + +func pinnedTopologyConfig() (types.DomainConfig, types.DomainStatus) { + config := types.DomainConfig{ + UUIDandVersion: types.UUIDandVersion{ + UUID: uuid.FromStringOrNil(DefaultUUID), Version: "1.0", + }, + VmConfig: types.VmConfig{ + Memory: 1024 * 1024, + VCpus: 4, + CPUsPinned: true, + CPUs: []uint32{2, 3, 6, 7}, + }, + } + status := types.DomainStatus{ + VMTopology: types.CPUTopology{Sockets: 1, Cores: 2, Threads: 2}, + OrderedCPUs: []uint32{2, 6, 3, 7}, + } + return config, status +} + +// Exposing the guest SMT topology is the point of whole-core pinning: a VM given +// two physical cores must see 2 cores x 2 threads, otherwise the guest scheduler +// treats siblings as independent cores and places two busy threads on one core. +func TestCreateDomConfigGuestSMTTopology(t *testing.T) { + config, status := pinnedTopologyConfig() + topo := status.VMTopology + // QEMU rejects an -smp whose topology does not multiply out to the cpus + // count, so the two can never be chosen independently. + if got := topo.Sockets * topo.Cores * topo.Threads; got != config.VCpus { + t.Fatalf("sockets*cores*threads = %d, want cpus = %d", got, config.VCpus) + } + want := `[smp-opts] + cpus = "4" + sockets = "1" + cores = "2" + threads = "2" +` + if got := renderDomConfig(t, kvmIntel, config, status, nil); !strings.Contains(got, want) { + t.Errorf("guest SMT topology missing from config:\nwant:\n%s\ngot:\n%s", want, got) + } +} + +// A VM with no computed topology keeps the legacy flat one vCPU per core layout. +func TestCreateDomConfigLegacySMTTopology(t *testing.T) { + config, _ := pinnedTopologyConfig() + want := `[smp-opts] + cpus = "4" + sockets = "1" + cores = "4" + threads = "1" +` + got := renderDomConfig(t, kvmIntel, config, types.DomainStatus{}, nil) + if !strings.Contains(got, want) { + t.Errorf("legacy smp topology missing from config:\nwant:\n%s\ngot:\n%s", want, got) + } +} + +// The virtio-blk iothread and the iothread0 object it references must appear +// together, and only for a VM whose emulator threads are moved to a +// housekeeping set. Every other VM keeps its block datapath on the main loop. +func TestCreateDomConfigBlockIOThread(t *testing.T) { + disks := []types.DiskStatus{ + {Format: zconfig.Format_QCOW2, FileLocation: "/foo/bar.qcow2", Devtype: "hdd"}, + } + tests := []struct { + name string + emulatorCPUs []uint32 + want bool + }{ + {"io_placement housekeeping", []uint32{0, 1}, true}, + {"io_placement dedicated", nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config, status := pinnedTopologyConfig() + status.EmulatorCPUs = tt.emulatorCPUs + got := renderDomConfig(t, kvmIntel, config, status, disks) + hasObject := strings.Contains(got, `[object "iothread0"]`) + hasDevice := strings.Contains(got, ` iothread = "iothread0"`) + if hasObject != tt.want || hasDevice != tt.want { + t.Errorf("iothread0 object=%v device=%v, want both %v:\n%s", + hasObject, hasDevice, tt.want, got) + } + }) + } +} + +// Legacy set-mask pinning and per-thread topology pinning are alternatives, so +// the thread-context object must disappear as soon as the allocator has produced +// an ordered CPU list. +func TestLegacyThreadContextArgs(t *testing.T) { + pinned := types.DomainConfig{ + VmConfig: types.VmConfig{CPUsPinned: true, CPUs: []uint32{2, 3, 6, 7}}, + } + tests := []struct { + name string + config types.DomainConfig + status types.DomainStatus + want []string + }{ + { + "legacy pinning emits the mask", + pinned, + types.DomainStatus{VmConfig: pinned.VmConfig}, + []string{"-object", "thread-context,id=tc1,cpu-affinity=2,cpu-affinity=3,cpu-affinity=6,cpu-affinity=7"}, + }, + { + "topology pinning takes over", + pinned, + types.DomainStatus{ + VmConfig: pinned.VmConfig, + OrderedCPUs: []uint32{2, 6, 3, 7}, + }, + nil, + }, + { + "unpinned VM gets nothing", + types.DomainConfig{}, + types.DomainStatus{}, + nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if diff := cmp.Diff(tt.want, legacyThreadContextArgs(tt.config, tt.status)); diff != "" { + t.Errorf("legacyThreadContextArgs() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/pillar/hypervisor/pinning.go b/pkg/pillar/hypervisor/pinning.go new file mode 100644 index 00000000000..b3c6d11cc71 --- /dev/null +++ b/pkg/pillar/hypervisor/pinning.go @@ -0,0 +1,133 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package hypervisor + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +// setThreadAffinity pins a single OS thread (tid) to the given CPU set. +func setThreadAffinity(tid int, cpus []uint32) error { + var set unix.CPUSet + set.Zero() + for _, c := range cpus { + set.Set(int(c)) + } + return unix.SchedSetaffinity(tid, &set) +} + +func qemuPid(domainName string) (int, error) { + data, err := os.ReadFile(kvmStateDir + domainName + "/pid") + if err != nil { + return 0, err + } + return strconv.Atoi(strings.TrimSpace(string(data))) +} + +// pinDomainThreads pins the domain's guest vCPU threads 1:1 (guest vCPU i -> +// ordered[i]) and, when emulator is non-empty (io_placement=housekeeping), +// pins every other QEMU thread to the emulator set. +// +// It is called from Start() at the one correct point in the containerd task +// lifecycle: after task Start has launched QEMU (paused via -S, so the vCPU +// threads exist and QMP is up) and BEFORE the QMP cont — i.e. after containerd +// has already written the cgroup cpuset, so our per-thread affinity is applied +// last and is not reset by the cpuset. ordered/emulator come straight from the +// domain's DomainStatus (bound into the Task via KvmContext.Task), so there is +// no ephemeral hand-off state and this is idempotent/safe to re-run on a boot +// retry. +// +// Every failure except a vanished thread is returned, so the caller keeps the +// guest paused instead of releasing it with a placement it does not have. +func (ctx KvmContext) pinDomainThreads(domainName, qmpFile string, ordered, emulator []uint32) error { + if len(ordered) == 0 { + return nil // not a topology-pinned domain + } + tids, err := QmpGetVcpuThreadIDs(qmpFile) + if err != nil { + return fmt.Errorf("query-cpus-fast: %w", err) + } + if len(tids) != len(ordered) { + return fmt.Errorf("vcpu count mismatch: qemu=%d ordered=%d", len(tids), len(ordered)) + } + vcpuTid := map[int]bool{} + for i, tid := range tids { + if tid <= 0 { + return fmt.Errorf("refusing to pin vcpu %d: invalid thread-id %d", i, tid) + } + vcpuTid[tid] = true + if err := setThreadAffinity(tid, []uint32{ordered[i]}); err != nil { + return fmt.Errorf("pin vcpu %d (tid %d) -> cpu %d: %w", i, tid, ordered[i], err) + } + } + // Log the full guest-vCPU -> thread -> host-CPU mapping, not just the CPU + // list. QEMU does not name its vCPU threads unless it is started with + // debug-threads=on, and the thread group also contains vhost_task helpers + // that look no different from the outside, so this mapping cannot be + // reconstructed from /proc afterwards -- QMP query-cpus-fast, which we just + // called, is the only authoritative source of it. Recording it here is what + // makes a pin verifiable after the fact, by a test or by a support engineer + // reading /proc//status. + var mapping strings.Builder + for i, tid := range tids { + if i > 0 { + mapping.WriteString(" ") + } + fmt.Fprintf(&mapping, "vcpu%d=tid%d@cpu%d", i, tid, ordered[i]) + } + logrus.Infof("CPU pinning: domain %s pinned %d vCPUs 1:1 to host CPUs %v [%s]", + domainName, len(tids), ordered, mapping.String()) + + if len(emulator) == 0 { + return nil // io_placement=dedicated: leave non-vCPU threads in the cgroup cpuset + } + pid, err := qemuPid(domainName) + if err != nil { + return fmt.Errorf("qemu pid: %w", err) + } + entries, err := os.ReadDir(fmt.Sprintf("/proc/%d/task", pid)) + if err != nil { + return fmt.Errorf("read qemu task dir: %w", err) + } + var pinned, failed int + var firstErr error + for _, e := range entries { + tid, err := strconv.Atoi(e.Name()) + if err != nil || vcpuTid[tid] { + continue + } + if err := setThreadAffinity(tid, emulator); err != nil { + // ESRCH is expected and harmless: QEMU has short-lived helper + // threads, and one of them can exit between reading /proc and the + // syscall. Every other errno is systemic rather than per-thread -- + // EINVAL means the mask holds no CPU inside the task's cgroup + // cpuset, so no thread is pinned at all -- and must fail the pin + // instead of leaving the emulator on the hot cores while the guest + // is released and the placement reported as achieved. + if errors.Is(err, unix.ESRCH) { + continue + } + failed++ + if firstErr == nil { + firstErr = fmt.Errorf("pin emulator thread %d -> cpus %v: %w", tid, emulator, err) + } + continue + } + pinned++ + } + if firstErr != nil { + return fmt.Errorf("%d of %d non-vCPU threads not pinned to %v: %w", + failed, failed+pinned, emulator, firstErr) + } + logrus.Infof("CPU pinning: domain %s pinned %d emulator/IO threads to host CPUs %v", + domainName, pinned, emulator) + return nil +} diff --git a/pkg/pillar/hypervisor/qmp.go b/pkg/pillar/hypervisor/qmp.go index dbd9c624ad0..d174dc70a02 100644 --- a/pkg/pillar/hypervisor/qmp.go +++ b/pkg/pillar/hypervisor/qmp.go @@ -171,6 +171,51 @@ func QmpExecDeviceAdd(socket, id string, busnum, devnum uint16) error { return err } +// QmpGetVcpuThreadIDs returns host thread IDs indexed by guest vCPU number +// via QMP query-cpus-fast. +func QmpGetVcpuThreadIDs(socket string) ([]int, error) { + raw, err := execRawCmd(socket, `{ "execute": "query-cpus-fast" }`, true) + if err != nil { + return nil, err + } + return parseVcpuThreadIDs(raw) +} + +func parseVcpuThreadIDs(raw []byte) ([]int, error) { + var resp struct { + Return []struct { + CPUIndex int `json:"cpu-index"` + ThreadID int `json:"thread-id"` + } `json:"return"` + } + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, fmt.Errorf("query-cpus-fast decode: %w", err) + } + n := len(resp.Return) + if n == 0 { + // Only ever called for a domain that has vCPUs to pin, so an empty reply + // is a broken monitor, not a valid answer. Say so here rather than let + // it resurface as a vCPU count mismatch in the caller. + return nil, fmt.Errorf("query-cpus-fast: no vCPUs reported") + } + out := make([]int, n) + seen := make([]bool, n) + for _, c := range resp.Return { + if c.CPUIndex < 0 || c.CPUIndex >= n { + return nil, fmt.Errorf("query-cpus-fast: cpu-index %d out of range [0,%d)", c.CPUIndex, n) + } + if seen[c.CPUIndex] { + return nil, fmt.Errorf("query-cpus-fast: duplicate cpu-index %d", c.CPUIndex) + } + if c.ThreadID <= 0 { + return nil, fmt.Errorf("query-cpus-fast: invalid thread-id %d for cpu-index %d", c.ThreadID, c.CPUIndex) + } + seen[c.CPUIndex] = true + out[c.CPUIndex] = c.ThreadID + } + return out, nil +} + // There is errors.Join(), but stupid Yetus has old golang // and complains with "Join not declared by package errors". // Use our own. diff --git a/pkg/pillar/hypervisor/qmp_test.go b/pkg/pillar/hypervisor/qmp_test.go index cd9289c84ef..ac311c79d1b 100644 --- a/pkg/pillar/hypervisor/qmp_test.go +++ b/pkg/pillar/hypervisor/qmp_test.go @@ -77,3 +77,71 @@ func TestBuildQMPCommandNoArguments(t *testing.T) { t.Errorf("got %q, want %q", got, want) } } + +// QEMU does not promise that query-cpus-fast lists CPUs in cpu-index order, and +// the result is consumed positionally (out[i] is guest vCPU i's thread), so the +// reply must be indexed rather than appended. The out-of-order rows below are +// what distinguishes the two. +func TestParseVcpuThreadIDs(t *testing.T) { + tests := []struct { + name string + raw string + want []int + }{ + { + "ascending", + `{"return":[{"cpu-index":0,"thread-id":101},{"cpu-index":1,"thread-id":102}]}`, + []int{101, 102}, + }, + { + "descending", + `{"return":[{"cpu-index":1,"thread-id":102},{"cpu-index":0,"thread-id":101}]}`, + []int{101, 102}, + }, + { + "shuffled 4 vcpus", + `{"return":[{"cpu-index":2,"thread-id":203},{"cpu-index":0,"thread-id":201},` + + `{"cpu-index":3,"thread-id":204},{"cpu-index":1,"thread-id":202}]}`, + []int{201, 202, 203, 204}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseVcpuThreadIDs([]byte(tt.raw)) + if err != nil { + t.Fatal(err) + } + if len(got) != len(tt.want) { + t.Fatalf("got %v, want %v", got, tt.want) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Errorf("vcpu %d: thread-id %d, want %d (got %v)", i, got[i], tt.want[i], got) + } + } + }) + } +} + +func TestParseVcpuThreadIDsErrors(t *testing.T) { + tests := []struct { + name string + raw string + }{ + {"out-of-range cpu-index", `{"return":[{"cpu-index":5,"thread-id":9}]}`}, + {"duplicate cpu-index", `{"return":[{"cpu-index":0,"thread-id":1},{"cpu-index":0,"thread-id":2}]}`}, + {"negative cpu-index", `{"return":[{"cpu-index":-1,"thread-id":1}]}`}, + {"zero thread-id", `{"return":[{"cpu-index":0,"thread-id":0}]}`}, + // A domain with no vCPU to pin never gets here, so an empty list is a + // broken monitor and must not be mistaken for a valid zero-vCPU answer. + {"empty return", `{"return":[]}`}, + {"malformed json", `{"return":[{"cpu-index":0,`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := parseVcpuThreadIDs([]byte(tt.raw)); err == nil { + t.Fatalf("expected an error, got %v", got) + } + }) + } +} From a04f075779ae3490f61d88276ffccff2bda6511d Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:00:47 +0000 Subject: [PATCH 05/15] hardware: report the CPU topology, caches and kernel isolation The CPU inventory emitted one entry per physical core with the core id in the field meant for a logical CPU id, no frequency and no topology. That is worse than incomplete: the ids were not the ones CPU affinities are expressed in, and the SMT structure -- the thing a consumer reasoning about CPU placement needs most -- was absent entirely. It now reports one entry per logical CPU carrying its socket, physical core, NUMA node and L3 domain, taken from the same topology discovery the allocator uses so the report and the behaviour cannot drift, plus base and maximum frequency where the kernel exposes them. Cache domains are reported with the set of CPUs sharing each one, which is what tells a consumer which workloads would contend for the same cache. The per-CPU sysfs views are collapsed into one entry per real cache instance. Kernel-level CPU isolation is reported separately as a node fact rather than a CPU one, and read from sysfs rather than parsed out of the command line, so it describes what the kernel is actually doing. The two differ when a parameter is malformed or capped, which is exactly when a consumer needs the truth. Topology discovery failing degrades to the previous flat listing instead of failing the whole inventory, which is still useful on a platform whose sysfs layout we cannot read. Signed-off-by: Mikhail Malyshev --- pkg/pillar/hardware/cpudetails.go | 219 ++++++++++++++++++++++ pkg/pillar/hardware/cpudetails_test.go | 242 +++++++++++++++++++++++++ pkg/pillar/hardware/inventory.go | 102 ++++++++++- pkg/pillar/hardware/inventory_test.go | 174 +++++++++++++++++- 4 files changed, 728 insertions(+), 9 deletions(-) create mode 100644 pkg/pillar/hardware/cpudetails.go create mode 100644 pkg/pillar/hardware/cpudetails_test.go diff --git a/pkg/pillar/hardware/cpudetails.go b/pkg/pillar/hardware/cpudetails.go new file mode 100644 index 00000000000..92b8046bbe0 --- /dev/null +++ b/pkg/pillar/hardware/cpudetails.go @@ -0,0 +1,219 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package hardware + +import ( + "os" + "path/filepath" + "slices" + "strconv" + "strings" +) + +// This file reads the CPU facts the hardware inventory reports beyond the +// topology coordinates that pkg/pillar/cputopology already discovers: cache +// domains, per-CPU frequency, and the kernel's effective CPU-isolation sets. +// +// sysfsRoot is a var so tests can point these readers at a fixture tree. +var sysfsRoot = "/sys/devices/system" + +// cacheDomain is one cache instance and the logical CPUs sharing it. +type cacheDomain struct { + // Level is the cache level as reported by sysfs (1, 2, 3, ...). + Level int + // Type is the sysfs cache type: "Data", "Instruction" or "Unified". It is + // part of a cache's identity because sysfs numbers ids per level, not per + // type - see readCacheDomains. Empty when sysfs omits the type file. + Type string + // ID is the platform's identifier for this cache instance. Caches of the + // same level and type with the same id are one cache. + ID uint32 + // SizeBytes is 0 when sysfs does not report a size. + SizeBytes uint64 + // CPUs are the logical CPUs sharing this cache instance, ascending. + CPUs []uint32 +} + +// readCacheDomains returns the distinct cache instances of the machine. +// +// Each CPU lists the caches it uses; instances are keyed by (level, type, id) so +// the per-CPU views collapse into one entry per real cache with the full set of +// CPUs that share it. That sharing is the point: it is what tells a consumer +// which workloads would contend for the same cache. sysfs numbers cache ids per +// level and not per type, so the L1 data and L1 instruction caches of one core +// are both "level 1, id N" and only the type file separates them - dropping the +// type from the key merges them into one bogus domain listing every CPU twice. +// Every level is reported; the caller decides which matter. +func readCacheDomains() []cacheDomain { + cpuDirs, err := filepath.Glob(filepath.Join(sysfsRoot, "cpu", "cpu[0-9]*")) + if err != nil || len(cpuDirs) == 0 { + return nil + } + + type key struct { + level int + cacheType string + id uint32 + } + domains := map[key]*cacheDomain{} + var order []key + + for _, cpuDir := range cpuDirs { + cpu, err := strconv.ParseUint( + strings.TrimPrefix(filepath.Base(cpuDir), "cpu"), 10, 32) + if err != nil { + continue + } + indexDirs, err := filepath.Glob(filepath.Join(cpuDir, "cache", "index[0-9]*")) + if err != nil { + continue + } + for _, indexDir := range indexDirs { + level, ok := readUintFile(filepath.Join(indexDir, "level")) + if !ok { + continue + } + cacheType := readTextFile(filepath.Join(indexDir, "type")) + id, ok := readUintFile(filepath.Join(indexDir, "id")) + if !ok { + // Some platforms omit the id; fall back to the shared-CPU list + // as the identity by using the lowest CPU in it. + shared := readCPUListFile(filepath.Join(indexDir, "shared_cpu_list")) + if len(shared) == 0 { + continue + } + id = uint64(shared[0]) + } + k := key{level: int(level), cacheType: cacheType, id: uint32(id)} + domain, seen := domains[k] + if !seen { + size, _ := readCacheSize(filepath.Join(indexDir, "size")) + domain = &cacheDomain{ + Level: int(level), + Type: cacheType, + ID: uint32(id), + SizeBytes: size, + } + domains[k] = domain + order = append(order, k) + } + domain.CPUs = append(domain.CPUs, uint32(cpu)) + } + } + + result := make([]cacheDomain, 0, len(order)) + for _, k := range order { + domain := domains[k] + slices.Sort(domain.CPUs) + result = append(result, *domain) + } + return result +} + +// cpuFrequencies returns a logical CPU's base and maximum frequency in kHz. +// Either is 0 when the kernel does not report it, which is normal on platforms +// without cpufreq. +func cpuFrequencies(cpu uint32) (baseKHz, maxKHz uint64) { + dir := filepath.Join(sysfsRoot, "cpu", "cpu"+strconv.FormatUint(uint64(cpu), 10), "cpufreq") + // base_frequency is the non-turbo nominal frequency and is the honest + // answer for "how fast is this CPU"; cpuinfo_max_freq includes turbo. + if value, ok := readUintFile(filepath.Join(dir, "base_frequency")); ok { + baseKHz = value + } + if value, ok := readUintFile(filepath.Join(dir, "cpuinfo_max_freq")); ok { + maxKHz = value + } + return baseKHz, maxKHz +} + +// IsolatedCPUSets returns the CPU sets the running kernel is treating specially. +// These are read from sysfs rather than parsed out of the kernel command line so +// they describe what the kernel is actually doing, which is what a consumer +// deciding where to place a latency-sensitive workload needs. +func IsolatedCPUSets() (isolated, nohzFull, rcuNocbs []uint32) { + cpuRoot := filepath.Join(sysfsRoot, "cpu") + return readCPUListFile(filepath.Join(cpuRoot, "isolated")), + readCPUListFile(filepath.Join(cpuRoot, "nohz_full")), + readCPUListFile(filepath.Join(cpuRoot, "rcu_nocbs")) +} + +// readTextFile reads a one-line sysfs attribute, yielding "" when absent. +func readTextFile(path string) string { + data, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +func readUintFile(path string) (uint64, bool) { + data, err := os.ReadFile(path) + if err != nil { + return 0, false + } + value, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64) + if err != nil { + return 0, false + } + return value, true +} + +// readCacheSize parses a sysfs cache size such as "32K" or "8192K" into bytes. +func readCacheSize(path string) (uint64, bool) { + data, err := os.ReadFile(path) + if err != nil { + return 0, false + } + text := strings.TrimSpace(string(data)) + multiplier := uint64(1) + switch { + case strings.HasSuffix(text, "K"): + multiplier, text = 1024, strings.TrimSuffix(text, "K") + case strings.HasSuffix(text, "M"): + multiplier, text = 1024*1024, strings.TrimSuffix(text, "M") + } + value, err := strconv.ParseUint(text, 10, 64) + if err != nil { + return 0, false + } + return value * multiplier, true +} + +// readCPUListFile reads a kernel CPU-list file. A missing file yields nothing: +// the kernel omits some of these entirely when the feature is unused. +func readCPUListFile(path string) []uint32 { + data, err := os.ReadFile(path) + if err != nil { + return nil + } + return parseCPUList(string(data)) +} + +// parseCPUList expands the kernel's CPU-list format ("0-2,5") into ids. +func parseCPUList(list string) []uint32 { + var cpus []uint32 + for _, part := range strings.Split(strings.TrimSpace(list), ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + loText, hiText, isRange := strings.Cut(part, "-") + lo, err := strconv.ParseUint(strings.TrimSpace(loText), 10, 32) + if err != nil { + continue + } + hi := lo + if isRange { + parsed, err := strconv.ParseUint(strings.TrimSpace(hiText), 10, 32) + if err != nil { + continue + } + hi = parsed + } + for cpu := lo; cpu <= hi; cpu++ { + cpus = append(cpus, uint32(cpu)) + } + } + return cpus +} diff --git a/pkg/pillar/hardware/cpudetails_test.go b/pkg/pillar/hardware/cpudetails_test.go new file mode 100644 index 00000000000..1a337ac6ff8 --- /dev/null +++ b/pkg/pillar/hardware/cpudetails_test.go @@ -0,0 +1,242 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package hardware + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +// writeFile creates path with content, making parent directories as needed. +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } +} + +// fakeSysfs builds a 4-CPU machine: two physical cores with two SMT threads +// each. Each core has its own L1d, L1i and L2; both share one L3 -- the layout +// that makes cache sharing worth reporting at all. As on a real host, the L1 +// data and instruction caches of a core carry the same level and the same id +// and differ only in their type file. +func fakeSysfs(t *testing.T) string { + t.Helper() + root := t.TempDir() + cpuRoot := filepath.Join(root, "cpu") + + type spec struct { + cpu int + coreID string // id of this core's private (L1, L2) caches + shared string // CPUs sharing them, i.e. the core's SMT threads + } + specs := []spec{ + {0, "0", "0-1"}, + {1, "0", "0-1"}, + {2, "1", "2-3"}, + {3, "1", "2-3"}, + } + for _, s := range specs { + dir := filepath.Join(cpuRoot, "cpu"+itoa(s.cpu)) + // L1 data and L1 instruction: per core, same level and same id. + writeFile(t, filepath.Join(dir, "cache/index0/level"), "1\n") + writeFile(t, filepath.Join(dir, "cache/index0/type"), "Data\n") + writeFile(t, filepath.Join(dir, "cache/index0/id"), s.coreID+"\n") + writeFile(t, filepath.Join(dir, "cache/index0/size"), "48K\n") + writeFile(t, filepath.Join(dir, "cache/index0/shared_cpu_list"), s.shared+"\n") + writeFile(t, filepath.Join(dir, "cache/index1/level"), "1\n") + writeFile(t, filepath.Join(dir, "cache/index1/type"), "Instruction\n") + writeFile(t, filepath.Join(dir, "cache/index1/id"), s.coreID+"\n") + writeFile(t, filepath.Join(dir, "cache/index1/size"), "32K\n") + writeFile(t, filepath.Join(dir, "cache/index1/shared_cpu_list"), s.shared+"\n") + // L2: per core. + writeFile(t, filepath.Join(dir, "cache/index2/level"), "2\n") + writeFile(t, filepath.Join(dir, "cache/index2/type"), "Unified\n") + writeFile(t, filepath.Join(dir, "cache/index2/id"), s.coreID+"\n") + writeFile(t, filepath.Join(dir, "cache/index2/size"), "1024K\n") + writeFile(t, filepath.Join(dir, "cache/index2/shared_cpu_list"), s.shared+"\n") + // L3: shared by all. + writeFile(t, filepath.Join(dir, "cache/index3/level"), "3\n") + writeFile(t, filepath.Join(dir, "cache/index3/type"), "Unified\n") + writeFile(t, filepath.Join(dir, "cache/index3/id"), "0\n") + writeFile(t, filepath.Join(dir, "cache/index3/size"), "8M\n") + writeFile(t, filepath.Join(dir, "cache/index3/shared_cpu_list"), "0-3\n") + // Frequency. + writeFile(t, filepath.Join(dir, "cpufreq/base_frequency"), "2400000\n") + writeFile(t, filepath.Join(dir, "cpufreq/cpuinfo_max_freq"), "4800000\n") + } + writeFile(t, filepath.Join(cpuRoot, "isolated"), "2-3\n") + writeFile(t, filepath.Join(cpuRoot, "nohz_full"), "2,3\n") + return root +} + +func itoa(i int) string { + return string(rune('0' + i)) +} + +func TestReadCacheDomains(t *testing.T) { + old := sysfsRoot + sysfsRoot = fakeSysfs(t) + defer func() { sysfsRoot = old }() + + domains := readCacheDomains() + + // Per-CPU views of the same cache must collapse into one entry each, and + // no further: an L1d and an L1i sharing a level and an id are two caches + // of different sizes, not one cache listing every CPU twice. + want := []cacheDomain{ + {Level: 1, Type: "Data", ID: 0, SizeBytes: 48 * 1024, CPUs: []uint32{0, 1}}, + {Level: 1, Type: "Instruction", ID: 0, SizeBytes: 32 * 1024, CPUs: []uint32{0, 1}}, + {Level: 1, Type: "Data", ID: 1, SizeBytes: 48 * 1024, CPUs: []uint32{2, 3}}, + {Level: 1, Type: "Instruction", ID: 1, SizeBytes: 32 * 1024, CPUs: []uint32{2, 3}}, + {Level: 2, Type: "Unified", ID: 0, SizeBytes: 1024 * 1024, CPUs: []uint32{0, 1}}, + {Level: 2, Type: "Unified", ID: 1, SizeBytes: 1024 * 1024, CPUs: []uint32{2, 3}}, + {Level: 3, Type: "Unified", ID: 0, SizeBytes: 8 * 1024 * 1024, CPUs: []uint32{0, 1, 2, 3}}, + } + if len(domains) != len(want) { + t.Fatalf("want %d cache domains (4x L1, 2x L2, 1x L3), got %d: %+v", + len(want), len(domains), domains) + } + for _, w := range want { + var got *cacheDomain + for i := range domains { + if domains[i].Level == w.Level && domains[i].Type == w.Type && domains[i].ID == w.ID { + if got != nil { + t.Fatalf("L%d %s id %d reported twice", w.Level, w.Type, w.ID) + } + got = &domains[i] + } + } + if got == nil { + t.Errorf("no L%d %s id %d domain reported, got %+v", w.Level, w.Type, w.ID, domains) + continue + } + if !reflect.DeepEqual(got.CPUs, w.CPUs) { + t.Errorf("L%d %s id %d CPUs = %v, want %v", w.Level, w.Type, w.ID, got.CPUs, w.CPUs) + } + if got.SizeBytes != w.SizeBytes { + t.Errorf("L%d %s id %d size = %d, want %d", + w.Level, w.Type, w.ID, got.SizeBytes, w.SizeBytes) + } + } +} + +// TestReadCacheDomains_NoTypeFile covers a platform whose cache indexes carry +// no type file: the caches must still collapse per (level, id) rather than +// disappearing or splitting per CPU. +func TestReadCacheDomains_NoTypeFile(t *testing.T) { + old := sysfsRoot + sysfsRoot = fakeSysfs(t) + defer func() { sysfsRoot = old }() + + typeFiles, err := filepath.Glob(filepath.Join(sysfsRoot, "cpu", "cpu*", "cache", "index*", "type")) + if err != nil || len(typeFiles) == 0 { + t.Fatalf("Glob(type files) = %d files, err %v", len(typeFiles), err) + } + for _, path := range typeFiles { + if err := os.Remove(path); err != nil { + t.Fatalf("Remove(%s): %v", path, err) + } + } + + // Without a type file the L1d and L1i of a core are indistinguishable and + // do merge -- unavoidably, and visibly as a doubled CPU list. + for _, domain := range readCacheDomains() { + if domain.Type != "" { + t.Errorf("L%d id %d Type = %q, want empty", domain.Level, domain.ID, domain.Type) + } + if domain.Level >= 2 && !reflect.DeepEqual(domain.CPUs, expectedCPUs(domain)) { + t.Errorf("L%d id %d CPUs = %v, want %v", + domain.Level, domain.ID, domain.CPUs, expectedCPUs(domain)) + } + } +} + +// expectedCPUs returns the CPUs the fixture's L2/L3 caches are shared by. +func expectedCPUs(domain cacheDomain) []uint32 { + if domain.Level == 3 { + return []uint32{0, 1, 2, 3} + } + return []uint32{domain.ID * 2, domain.ID*2 + 1} +} + +func TestCPUFrequencies(t *testing.T) { + old := sysfsRoot + sysfsRoot = fakeSysfs(t) + defer func() { sysfsRoot = old }() + + base, max := cpuFrequencies(0) + if base != 2400000 || max != 4800000 { + t.Errorf("frequencies = %d/%d kHz, want 2400000/4800000", base, max) + } + + // A platform without cpufreq must report zeros rather than failing: the + // rest of the inventory is still worth sending. + base, max = cpuFrequencies(99) + if base != 0 || max != 0 { + t.Errorf("missing cpufreq should report 0/0, got %d/%d", base, max) + } +} + +func TestIsolatedCPUSets(t *testing.T) { + old := sysfsRoot + sysfsRoot = fakeSysfs(t) + defer func() { sysfsRoot = old }() + + isolated, nohzFull, rcuNocbs := IsolatedCPUSets() + if !reflect.DeepEqual(isolated, []uint32{2, 3}) { + t.Errorf("isolated = %v, want [2 3]", isolated) + } + if !reflect.DeepEqual(nohzFull, []uint32{2, 3}) { + t.Errorf("nohz_full = %v, want [2 3]", nohzFull) + } + // rcu_nocbs is absent from the fixture, as it is on a stock kernel. + if len(rcuNocbs) != 0 { + t.Errorf("rcu_nocbs = %v, want empty when the file is absent", rcuNocbs) + } +} + +func TestParseCPUList(t *testing.T) { + tests := []struct { + in string + want []uint32 + }{ + {"", nil}, + {"\n", nil}, + {"3", []uint32{3}}, + {"0-3", []uint32{0, 1, 2, 3}}, + {"0-2,5", []uint32{0, 1, 2, 5}}, + {"2,3\n", []uint32{2, 3}}, + {"1-2,7-8", []uint32{1, 2, 7, 8}}, + } + for _, tt := range tests { + if got := parseCPUList(tt.in); !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseCPUList(%q) = %v, want %v", tt.in, got, tt.want) + } + } +} + +func TestReadCacheSize(t *testing.T) { + dir := t.TempDir() + for _, tt := range []struct { + content string + want uint64 + }{ + {"32K", 32 * 1024}, + {"8M", 8 * 1024 * 1024}, + {"512", 512}, + } { + path := filepath.Join(dir, "size") + writeFile(t, path, tt.content) + got, ok := readCacheSize(path) + if !ok || got != tt.want { + t.Errorf("readCacheSize(%q) = %d (ok=%v), want %d", tt.content, got, ok, tt.want) + } + } +} diff --git a/pkg/pillar/hardware/inventory.go b/pkg/pillar/hardware/inventory.go index fe11f43e130..9e4a3c503ba 100644 --- a/pkg/pillar/hardware/inventory.go +++ b/pkg/pillar/hardware/inventory.go @@ -17,7 +17,9 @@ import ( pcitypes "github.com/jaypipes/pcidb/types" "github.com/lf-edge/eve-api/go/info" "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/cputopology" "github.com/lf-edge/eve/pkg/pillar/evetpm" + "github.com/sirupsen/logrus" "github.com/zededa/ghw" "github.com/zededa/ghw/pkg/can" "github.com/zededa/ghw/pkg/option" @@ -36,6 +38,7 @@ func GetInventoryInfo(log *base.LogObject) (*info.HardwareInventory, error) { inventory.CanDevices, errs["CAN"] = getCANDevices() inventory.Bios, errs["BIOS"] = getBIOSInfo() inventory.CpuInfo, errs["CPU"] = getCPUInfo() + inventory.NodeCapabilities = getNodeCapabilities() inventory.TotalMemoryBytes, errs["Memory"] = getMemoryBytes() inventory.TotalStorageBytes, errs["Storage"] = getStorageBytes() inventory.WatchdogPresent, errs["Watchdog"] = watchdogPresent() @@ -425,20 +428,103 @@ func getCPUInfo() (*info.CPUInfo, error) { if err != nil { return nil, err } - cpuInfoProto := &info.CPUInfo{} - for _, proc := range cpuInfo.Processors { - for _, core := range proc.Cores { - c := info.CPU{ - Model: proc.Model, - Vendor: proc.Vendor, - Id: uint32(core.ID), + // Model and vendor are per-package; every logical CPU of a package reports + // its package's values. + model, vendor := "", "" + if len(cpuInfo.Processors) > 0 { + model = cpuInfo.Processors[0].Model + vendor = cpuInfo.Processors[0].Vendor + } + + cpuInfoProto := &info.CPUInfo{ + // Phase 1 reports no RDT support. The fields exist so a controller can + // gate on them once cache and memory-bandwidth allocation are + // implemented; claiming them now would be wrong. + Capabilities: &info.CPUCapabilities{}, + } + + // One entry per *logical* CPU, each carrying its position in the topology. + // Reporting one entry per physical core (as this did before) loses the + // SMT structure entirely, which is exactly what a consumer reasoning about + // CPU placement needs -- and it made the reported ids ambiguous, since a + // core id is not a schedulable CPU id. + topo, terr := cputopology.DiscoverTopology() + if terr != nil { + // Without topology we can still report the CPUs themselves. Degrading + // here keeps the rest of the inventory useful on a platform whose sysfs + // layout we cannot read. + logrus.Warnf("getCPUInfo: topology discovery failed: %v", terr) + for _, proc := range cpuInfo.Processors { + for _, core := range proc.Cores { + cpuInfoProto.Cpus = append(cpuInfoProto.Cpus, &info.CPU{ + Model: proc.Model, + Vendor: proc.Vendor, + Id: uint32(core.ID), + }) } - cpuInfoProto.Cpus = append(cpuInfoProto.Cpus, &c) } + return cpuInfoProto, nil + } + + for _, core := range topo.Cores { + for _, lcpu := range core.Siblings { + baseKHz, maxKHz := cpuFrequencies(uint32(lcpu)) + cpuInfoProto.Cpus = append(cpuInfoProto.Cpus, &info.CPU{ + Model: model, + Vendor: vendor, + Id: uint32(lcpu), + SocketId: uint32(core.Socket), + CoreId: uint32(core.CoreID), + NumaNode: uint32(core.NUMA), + L3Id: uint32(core.L3ID), + BaseFreqKhz: baseKHz, + MaxFreqKhz: maxKHz, + }) + } + } + + for _, domain := range readCacheDomains() { + level := cacheLevelToProto(domain.Level) + if level == info.CacheLevel_CACHE_LEVEL_UNSPECIFIED { + continue + } + cpuInfoProto.Caches = append(cpuInfoProto.Caches, &info.CacheDomain{ + Level: level, + Id: domain.ID, + SizeBytes: domain.SizeBytes, + CpuIds: domain.CPUs, + }) } return cpuInfoProto, nil } +// cacheLevelToProto maps a sysfs cache level onto the reported enum. Level 1 is +// split into instruction and data caches which sysfs distinguishes by type; the +// inventory reports the levels that matter for contention between workloads and +// leaves the rest unspecified. +func cacheLevelToProto(level int) info.CacheLevel { + switch level { + case 2: + return info.CacheLevel_CACHE_LEVEL_L2 + case 3: + return info.CacheLevel_CACHE_LEVEL_L3 + default: + return info.CacheLevel_CACHE_LEVEL_UNSPECIFIED + } +} + +// getNodeCapabilities reports the kernel-level CPU isolation actually in effect. +// These are node facts, not CPU facts, and deliberately describe what the kernel +// is doing rather than what its command line asked for. +func getNodeCapabilities() *info.NodeCapabilities { + isolated, nohzFull, rcuNocbs := IsolatedCPUSets() + return &info.NodeCapabilities{ + IsolatedCpuIds: isolated, + NohzFullCpuIds: nohzFull, + RcuNocbsCpuIds: rcuNocbs, + } +} + func getMemoryBytes() (uint64, error) { memInfo, err := ghw.Memory(option.WithDisableTools()) if err != nil { diff --git a/pkg/pillar/hardware/inventory_test.go b/pkg/pillar/hardware/inventory_test.go index 6edd6f9da35..8df33e566e8 100644 --- a/pkg/pillar/hardware/inventory_test.go +++ b/pkg/pillar/hardware/inventory_test.go @@ -7,13 +7,23 @@ import ( "encoding/json" "errors" "os" + "path/filepath" + "strconv" + "strings" "testing" + "github.com/lf-edge/eve-api/go/info" "github.com/lf-edge/eve/pkg/pillar/agentlog" ) +// The assertions below run against the real /sys of whatever host the test runs +// on, so they are written as properties that hold on any Linux machine rather +// than against this machine's topology: one entry per online logical CPU, ids +// that agree with sysfs, and cache domains that name real CPUs. Reporting the +// wrong socket/NUMA/L3 id, or a core id where a schedulable CPU id belongs, is +// exactly the kind of mistake that makes a placement decision taken from this +// inventory land on the wrong CPUs. func TestCreateInventory(t *testing.T) { - _, log := agentlog.Init("someAgent") inventory, err := GetInventoryInfo(log) @@ -27,4 +37,166 @@ func TestCreateInventory(t *testing.T) { t.Fatalf("could not create json: %v", err) } t.Log(string(bytes)) + + // The kernel isolation facts are a node-level answer that is always + // available: empty sets mean "the kernel isolates nothing", which a + // controller must be able to tell from "this build does not report it". + if inventory.NodeCapabilities == nil { + t.Error("NodeCapabilities is nil, so a controller cannot tell an " + + "un-isolated node from an EVE that does not report isolation") + } + if inventory.CpuInfo == nil { + t.Fatal("CpuInfo is nil") + } + + online := onlineCPUsFromSysfs(t) + if len(online) == 0 { + t.Skip("cannot read the online CPU list from sysfs; " + + "the per-CPU assertions below have nothing to compare against") + } + + reported := map[uint32]*info.CPU{} + for _, cpu := range inventory.CpuInfo.Cpus { + if _, dup := reported[cpu.Id]; dup { + t.Errorf("CPU %d reported twice", cpu.Id) + } + reported[cpu.Id] = cpu + } + // One entry per *logical* CPU. One entry per physical core instead would + // hide the SMT structure and report core ids as if they were CPU ids. + if len(reported) != len(online) { + t.Errorf("reported %d CPUs, want %d online logical CPUs (%v)", + len(reported), len(online), online) + } + for _, cpu := range online { + got, ok := reported[cpu] + if !ok { + t.Errorf("online cpu%d is missing from the inventory", cpu) + continue + } + if want, ok := sysfsCPUUint(t, cpu, "topology/core_id"); ok && got.CoreId != want { + t.Errorf("cpu%d core_id reported as %d, sysfs says %d", + cpu, got.CoreId, want) + } + // Absent on single-socket systems, where socket 0 is the right answer. + want, present := sysfsCPUUint(t, cpu, "topology/physical_package_id") + if !present { + want = 0 + } + if got.SocketId != want { + t.Errorf("cpu%d socket reported as %d, sysfs says %d", + cpu, got.SocketId, want) + } + // A NUMA node id no node directory corresponds to is fabricated + // locality, which a single-NUMA-node placement request would trust. + nodeDir := filepath.Join("/sys/devices/system/node", + "node"+strconv.FormatUint(uint64(got.NumaNode), 10)) + if _, err := os.Stat("/sys/devices/system/node"); err == nil { + if _, err := os.Stat(nodeDir); err != nil { + t.Errorf("cpu%d reported on NUMA node %d, but %s does not exist", + cpu, got.NumaNode, nodeDir) + } + } else if got.NumaNode != 0 { + t.Errorf("cpu%d reported on NUMA node %d on a kernel that exposes "+ + "no NUMA information", cpu, got.NumaNode) + } + } + + for _, cache := range inventory.CpuInfo.Caches { + switch cache.Level { + case info.CacheLevel_CACHE_LEVEL_L2, info.CacheLevel_CACHE_LEVEL_L3: + default: + t.Errorf("cache domain %d reported at level %v; only the levels "+ + "workloads contend over are meant to be reported", + cache.Id, cache.Level) + } + if len(cache.CpuIds) == 0 { + t.Errorf("cache domain %d (%v) lists no CPUs, so nothing can be "+ + "said about which workloads share it", cache.Id, cache.Level) + } + for _, id := range cache.CpuIds { + dir := filepath.Join("/sys/devices/system/cpu", + "cpu"+strconv.FormatUint(uint64(id), 10)) + if _, err := os.Stat(dir); err != nil { + t.Errorf("cache domain %d (%v) lists cpu%d, which does not "+ + "exist: these must be logical CPU ids, not core ids", + cache.Id, cache.Level, id) + } + } + } + if sysfsHasSharedCache(online[0]) && len(inventory.CpuInfo.Caches) == 0 { + t.Errorf("no cache domains reported although sysfs exposes an L2/L3 "+ + "cache for cpu%d", online[0]) + } +} + +// onlineCPUsFromSysfs reads the kernel's own list of online logical CPUs, which +// is the independent answer the inventory is checked against. It returns nil if +// sysfs does not expose it. +func onlineCPUsFromSysfs(t *testing.T) []uint32 { + t.Helper() + data, err := os.ReadFile("/sys/devices/system/cpu/online") + if err != nil { + return nil + } + var cpus []uint32 + for _, part := range strings.Split(strings.TrimSpace(string(data)), ",") { + if part == "" { + continue + } + lo, hi, isRange := strings.Cut(part, "-") + first, err := strconv.ParseUint(lo, 10, 32) + if err != nil { + t.Fatalf("unparseable cpu range %q: %v", part, err) + } + last := first + if isRange { + last, err = strconv.ParseUint(hi, 10, 32) + if err != nil { + t.Fatalf("unparseable cpu range %q: %v", part, err) + } + } + for cpu := first; cpu <= last; cpu++ { + cpus = append(cpus, uint32(cpu)) + } + } + return cpus +} + +// sysfsCPUUint reads a per-CPU sysfs attribute, reporting whether it exists. +func sysfsCPUUint(t *testing.T, cpu uint32, attr string) (uint32, bool) { + t.Helper() + path := filepath.Join("/sys/devices/system/cpu", + "cpu"+strconv.FormatUint(uint64(cpu), 10), attr) + data, err := os.ReadFile(path) + if err != nil { + return 0, false + } + value, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 32) + if err != nil { + return 0, false + } + return uint32(value), true +} + +// sysfsHasSharedCache reports whether sysfs describes an L2 or L3 cache for the +// given CPU, i.e. whether the inventory is expected to report cache domains at +// all on this host. +func sysfsHasSharedCache(cpu uint32) bool { + indexes, err := filepath.Glob(filepath.Join("/sys/devices/system/cpu", + "cpu"+strconv.FormatUint(uint64(cpu), 10), "cache", "index[0-9]*")) + if err != nil { + return false + } + for _, index := range indexes { + data, err := os.ReadFile(filepath.Join(index, "level")) + if err != nil { + continue + } + switch strings.TrimSpace(string(data)) { + case "2", "3": + return true + } + } + return false } From 1a79f73fb7c89473ed5922b55b802b36697319b8 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:01:10 +0000 Subject: [PATCH 06/15] zedagent: parse the CPU placement policy and report what the node can do Maps the VmConfig CPU placement fields onto the device-internal intent and derives CPUsPinned from it. Two properties matter. An unrecognised enum value from a newer controller degrades to "no preference" rather than being rejected, which is safe because a controller is expected to gate on the capability reports below. And a dedicated policy is self-sufficient: it implies pinning on its own, so a workload no longer has to set the legacy pin_cpu flag as well for its CPUs to actually be pinned. With no policy sent, pin_cpu decides exactly as before. Advertises API_CAPABILITY_CPU_PLACEMENT_POLICY. Until this is reported a controller has no way to know the device honours the placement fields at all, which is precisely the failure the existing enforced-network-interface-order capability guards against, and it is what makes the fail-open behaviour above sound. Reports the node's CPU pool utilization on device info, per pool, with both the CPU sets and the whole-core counts, so a controller can answer "will this fit?" before a deploy and explain a shortage after one. This is dynamic state, so it rides the change-driven message rather than the cached hardware inventory. Surfaces a sub-optimal placement per application as a non-fatal advisory. It is converted to an ErrorInfo only at the wire, and never placed in the status error fields, because those are read as fatal in several places and a workload whose placement is merely improvable must not be torn down for it. Signed-off-by: Mikhail Malyshev --- .../cmd/zedagent/cpuplacementinfo_test.go | 137 ++++++++++ pkg/pillar/cmd/zedagent/handlemetrics.go | 39 +++ pkg/pillar/cmd/zedagent/parseconfig.go | 6 +- pkg/pillar/cmd/zedagent/parseconfig_test.go | 84 ++++++ pkg/pillar/cmd/zedagent/parsecpuplacement.go | 101 +++++++ .../cmd/zedagent/parsecpuplacement_test.go | 248 ++++++++++++++++++ pkg/pillar/cmd/zedagent/reportinfo.go | 76 +++++- pkg/pillar/cmd/zedagent/reportinfo_test.go | 188 +++++++++++++ pkg/pillar/cmd/zedagent/zedagent.go | 20 ++ 9 files changed, 897 insertions(+), 2 deletions(-) create mode 100644 pkg/pillar/cmd/zedagent/cpuplacementinfo_test.go create mode 100644 pkg/pillar/cmd/zedagent/parsecpuplacement.go create mode 100644 pkg/pillar/cmd/zedagent/parsecpuplacement_test.go create mode 100644 pkg/pillar/cmd/zedagent/reportinfo_test.go diff --git a/pkg/pillar/cmd/zedagent/cpuplacementinfo_test.go b/pkg/pillar/cmd/zedagent/cpuplacementinfo_test.go new file mode 100644 index 00000000000..f521d919fb2 --- /dev/null +++ b/pkg/pillar/cmd/zedagent/cpuplacementinfo_test.go @@ -0,0 +1,137 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package zedagent + +import ( + "testing" + "time" + + "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +func repackableApp() *types.AppInstanceStatus { + status := &types.AppInstanceStatus{ + DisplayName: "vpp", + State: types.RUNNING, + Activated: true, + PlacementQuality: types.CPUPlacementQualityNeedsRepack, + BootTime: time.Date(2026, 8, 8, 12, 0, 0, 0, time.UTC), + } + return status +} + +// A workload that would be better placed after a repack must reach the +// controller -- otherwise the whole "the device says which apps are affected, +// the controller decides whether a restart is worth it" loop has no input. +func TestCPUPlacementAdvisory_ReportsNeedsRepack(t *testing.T) { + errInfo := cpuPlacementAdvisory(repackableApp()) + if errInfo == nil { + t.Fatal("a workload needing a repack must be reported") + } + if errInfo.ErrorCode != types.ErrorCodeCPUPlacementNeedsRepack { + t.Errorf("error_code %q, want %q", errInfo.ErrorCode, + types.ErrorCodeCPUPlacementNeedsRepack) + } + if errInfo.Severity != info.Severity_SEVERITY_WARNING { + t.Errorf("severity %v, want WARNING: a sub-optimally placed workload is "+ + "running normally and must not be reported as failed", errInfo.Severity) + } + if errInfo.Timestamp == nil { + t.Error("an ErrorInfo without a timestamp is dropped on the wire") + } +} + +// The advisory must not make the app look broken: it is carried alongside the +// app's real error state, never inside it, so nothing that tests HasError() +// starts treating the workload as failed. +func TestCPUPlacementAdvisory_DoesNotFailTheApp(t *testing.T) { + status := repackableApp() + + if errInfo := cpuPlacementAdvisory(status); errInfo == nil { + t.Fatal("expected an advisory") + } + if status.HasError() { + t.Error("the advisory must not put the app into an error state") + } + if status.State != types.RUNNING { + t.Errorf("state %v, want RUNNING", status.State) + } +} + +// Everything else is silent: an optimally placed workload, one whose placement +// was never evaluated, and one that is not pinned at all. +func TestCPUPlacementAdvisory_SilentOtherwise(t *testing.T) { + for _, quality := range []types.CPUPlacementQuality{ + types.CPUPlacementQualityUnspecified, + types.CPUPlacementQualityOptimal, + } { + status := repackableApp() + status.PlacementQuality = quality + if errInfo := cpuPlacementAdvisory(status); errInfo != nil { + t.Errorf("quality %v must not be reported, got %+v", quality, errInfo) + } + } +} + +// The timestamp has to be stable, or every periodic info message would look +// like a fresh occurrence of the same condition. +func TestCPUPlacementAdvisory_TimestampIsStable(t *testing.T) { + status := repackableApp() + first := cpuPlacementAdvisory(status) + second := cpuPlacementAdvisory(status) + if !first.Timestamp.AsTime().Equal(second.Timestamp.AsTime()) { + t.Errorf("timestamp moved between reports: %v then %v", + first.Timestamp.AsTime(), second.Timestamp.AsTime()) + } + if !first.Timestamp.AsTime().Equal(status.BootTime) { + t.Errorf("timestamp %v, want the boot the placement was decided at (%v)", + first.Timestamp.AsTime(), status.BootTime) + } +} + +// TestCPUPlacementAdvisory_RidesAlongsideARealAppError is the premise of the whole +// advisory design: a workload whose placement is merely improvable must never be +// torn down for it, so the advisory is a second entry on ZInfoApp.app_err rather +// than a replacement for the app's real error. +// +// The append itself lives in PublishAppInfoToZedCloud, which needs the full +// zedagentContext (app IPs, network instances, the send queue) to call; this +// covers the two producers whose output it concatenates -- that both yield an +// entry for the same status, and that each keeps its own severity and code. +func TestCPUPlacementAdvisory_RidesAlongsideARealAppError(t *testing.T) { + status := repackableApp() + status.SetErrorWithSourceAndDescription(types.ErrorDescription{ + Error: "domain failed to attach a passthrough device", + ErrorCode: "domain.adapter.failed", + ErrorSeverity: types.ErrorSeverityError, + ErrorTime: time.Date(2026, 8, 8, 12, 5, 0, 0, time.UTC), + }, types.DomainStatus{}) + + appErr := status.ErrorAndTimeWithSource.ErrorDescription.ToProto() + advisory := cpuPlacementAdvisory(status) + if appErr == nil { + t.Fatal("the app's own error must still be reported") + } + if advisory == nil { + t.Fatal("an app that already has an error must still get the placement " + + "advisory: suppressing it loses the only signal a repack is possible") + } + + if appErr.Severity != info.Severity_SEVERITY_ERROR { + t.Errorf("the app's own error was reported as %v, want ERROR: the advisory "+ + "must not downgrade a genuine failure", appErr.Severity) + } + if advisory.Severity != info.Severity_SEVERITY_WARNING { + t.Errorf("advisory severity %v, want WARNING", advisory.Severity) + } + if appErr.ErrorCode == advisory.ErrorCode { + t.Errorf("both entries carry error_code %q, so a controller cannot tell "+ + "the failure from the advisory", appErr.ErrorCode) + } + if advisory.ErrorCode != types.ErrorCodeCPUPlacementNeedsRepack { + t.Errorf("advisory error_code %q, want %q", advisory.ErrorCode, + types.ErrorCodeCPUPlacementNeedsRepack) + } +} diff --git a/pkg/pillar/cmd/zedagent/handlemetrics.go b/pkg/pillar/cmd/zedagent/handlemetrics.go index 2259bc44923..637e85315c3 100644 --- a/pkg/pillar/cmd/zedagent/handlemetrics.go +++ b/pkg/pillar/cmd/zedagent/handlemetrics.go @@ -1132,6 +1132,42 @@ func encodeProxyStatus(proxyConfig *types.ProxyConfig) *info.ProxyStatus { return status } +// cpuPlacementAdvisory reports a running workload whose CPU placement could be +// improved, as a non-fatal advisory on the app's info message. +// +// It rides the app's error list because that is the only per-app channel the +// API has for a typed, machine-parseable condition. It is emitted at WARNING +// severity with the app left in whatever state it is really in -- normally +// RUNNING -- because nothing is broken: the workload has exactly the CPUs it +// asked for, just not in the arrangement that restarting its neighbours would +// achieve. Whether that restart is worth its disruption is the controller's +// call, and it can only make it if the device says so. +// +// Deliberately not routed through AppInstanceStatus's error fields: HasError() +// is severity-blind and several call sites treat any error as a failure, which +// would turn a merely sub-optimal placement into a refused workload. +func cpuPlacementAdvisory(aiStatus *types.AppInstanceStatus) *info.ErrorInfo { + if aiStatus.PlacementQuality != types.CPUPlacementQualityNeedsRepack { + return nil + } + // Timestamp the advisory with the boot the placement was decided at, so it + // does not appear to change on every info message. + errorTime := aiStatus.BootTime + if errorTime.IsZero() { + errorTime = time.Now() + } + description := types.ErrorDescription{ + Error: fmt.Sprintf("%s is running, but on a CPU placement worse than the "+ + "device could achieve by moving the workloads around it", + aiStatus.DisplayName), + ErrorCode: types.ErrorCodeCPUPlacementNeedsRepack, + ErrorSeverity: types.ErrorSeverityWarning, + ErrorTime: errorTime, + ErrorRetryCondition: "Cleared once the workload is placed optimally, which needs the pinned workloads restarted together", + } + return description.ToProto() +} + // This function is called per change, hence needs to try over all management ports // When aiStatus is nil it means a delete and we send a message // containing only the UUID to inform zedcloud about the delete. @@ -1169,6 +1205,9 @@ func PublishAppInfoToZedCloud(ctx *zedagentContext, uuid string, errInfo := aiStatus.ErrorAndTimeWithSource.ErrorDescription.ToProto() ReportAppInfo.AppErr = append(ReportAppInfo.AppErr, errInfo) } + if errInfo := cpuPlacementAdvisory(aiStatus); errInfo != nil { + ReportAppInfo.AppErr = append(ReportAppInfo.AppErr, errInfo) + } if aiStatus.BootTime.IsZero() { // If never booted diff --git a/pkg/pillar/cmd/zedagent/parseconfig.go b/pkg/pillar/cmd/zedagent/parseconfig.go index 71c29d15877..3f925c17e91 100644 --- a/pkg/pillar/cmd/zedagent/parseconfig.go +++ b/pkg/pillar/cmd/zedagent/parseconfig.go @@ -722,7 +722,11 @@ func parseAppInstanceConfig(getconfigCtx *getconfigContext, appInstance.Delay = time.Duration(cfgApp.StartDelayInSeconds) * time.Second appInstance.Service = cfgApp.Service appInstance.CloudInitVersion = cfgApp.CloudInitVersion - appInstance.FixedResources.CPUsPinned = cfgApp.Fixedresources.PinCpu + appInstance.FixedResources.CPUPlacement = + parseCPUPlacementPolicy(cfgApp.Fixedresources) + appInstance.FixedResources.CPUsPinned = + cpusPinnedFromPolicy(appInstance.FixedResources.CPUPlacement, + cfgApp.Fixedresources.PinCpu) appInstance.FixedResources.EnableOemWinLicenseKey = cfgApp.Fixedresources.EnableOemWinLicenseKey appInstance.FixedResources.DisableVirtualTPM = cfgApp.Fixedresources.DisableVtpm diff --git a/pkg/pillar/cmd/zedagent/parseconfig_test.go b/pkg/pillar/cmd/zedagent/parseconfig_test.go index edba5295305..f6347e20e24 100644 --- a/pkg/pillar/cmd/zedagent/parseconfig_test.go +++ b/pkg/pillar/cmd/zedagent/parseconfig_test.go @@ -2851,3 +2851,87 @@ func marshalJSONIgnoreOmitEmpty(u *config.EdgeDevConfig) ([]byte, error) { newType := reflect.StructOf(fields) return json.Marshal(value.Convert(newType).Interface()) } + +// TestParseAppInstanceConfig_StoresCPUPlacement proves the placement fields are +// actually parsed and stored, not merely parseable: the mappers are unit-tested +// on their own, but nothing else asserts that parseAppInstanceConfig calls them +// and puts the result on the published AppInstanceConfig, which is the only way +// the intent reaches zedmanager and domainmgr. +func TestParseAppInstanceConfig_StoresCPUPlacement(t *testing.T) { + g := NewGomegaWithT(t) + getconfigCtx, _ := newFuzzGetConfigCtx(t) + + const appUUID = "6ba7b810-9dad-11d1-80b4-00c04fd430a7" + edgeConfig := &zconfig.EdgeDevConfig{ + Id: &zconfig.UUIDandVersion{ + Uuid: "6ba7b810-9dad-11d1-80b4-00c04fd430c8", + Version: "1", + }, + Apps: []*zconfig.AppInstanceConfig{{ + Uuidandversion: &zconfig.UUIDandVersion{Uuid: appUUID, Version: "1"}, + Displayname: "vpp", + Fixedresources: &zconfig.VmConfig{ + Vcpus: 4, + Memory: 1024, + CpuPolicy: zconfig.CpuPolicy_CPU_POLICY_DEDICATED, + FullPcpusOnly: true, + ThreadsPerCore: 1, + NumaPolicy: zconfig.NumaPolicy_NUMA_POLICY_SINGLE_NUMA_NODE, + IoPlacement: zconfig.IoPlacement_IO_PLACEMENT_HOUSEKEEPING, + IsolationTier: zconfig.IsolationTier_ISOLATION_TIER_SOFT, + DisruptionPolicy: zconfig.DisruptionPolicy_DISRUPTION_POLICY_PROTECT, + // Deliberately not set: a dedicated policy must pin on its own. + PinCpu: false, + }, + }}, + } + + appinstancePrevConfigHash = nil + parseAppInstanceConfig(getconfigCtx, edgeConfig) + + item, err := getconfigCtx.pubAppInstanceConfig.Get(appUUID) + g.Expect(err).To(BeNil()) + appConfig, ok := item.(types.AppInstanceConfig) + g.Expect(ok).To(BeTrue()) + g.Expect(appConfig.FixedResources.CPUPlacement).To(Equal(types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, + FullPCPUsOnly: true, + ThreadsPerCore: 1, + NUMAPolicy: types.CPUNUMAPolicySingleNode, + IOPlacement: types.CPUIOPlacementHousekeeping, + IsolationTier: types.CPUIsolationTierSoft, + DisruptionPolicy: types.CPUDisruptionPolicyProtect, + })) + g.Expect(appConfig.FixedResources.CPUsPinned).To(BeTrue(), + "a dedicated policy must set the legacy pin flag, or nothing downstream pins") +} + +// TestParseAppInstanceConfig_KeepsLegacyPinCpu is the other half: a controller +// that sends no placement policy must keep behaving exactly as before, with +// pin_cpu alone deciding. +func TestParseAppInstanceConfig_KeepsLegacyPinCpu(t *testing.T) { + g := NewGomegaWithT(t) + getconfigCtx, _ := newFuzzGetConfigCtx(t) + + const appUUID = "6ba7b810-9dad-11d1-80b4-00c04fd430a8" + edgeConfig := &zconfig.EdgeDevConfig{ + Id: &zconfig.UUIDandVersion{ + Uuid: "6ba7b810-9dad-11d1-80b4-00c04fd430c8", + Version: "1", + }, + Apps: []*zconfig.AppInstanceConfig{{ + Uuidandversion: &zconfig.UUIDandVersion{Uuid: appUUID, Version: "1"}, + Displayname: "legacy", + Fixedresources: &zconfig.VmConfig{Vcpus: 2, Memory: 512, PinCpu: true}, + }}, + } + + appinstancePrevConfigHash = nil + parseAppInstanceConfig(getconfigCtx, edgeConfig) + + item, err := getconfigCtx.pubAppInstanceConfig.Get(appUUID) + g.Expect(err).To(BeNil()) + appConfig := item.(types.AppInstanceConfig) + g.Expect(appConfig.FixedResources.CPUPlacement).To(Equal(types.CPUPlacementPolicy{})) + g.Expect(appConfig.FixedResources.CPUsPinned).To(BeTrue()) +} diff --git a/pkg/pillar/cmd/zedagent/parsecpuplacement.go b/pkg/pillar/cmd/zedagent/parsecpuplacement.go new file mode 100644 index 00000000000..5f261f5341e --- /dev/null +++ b/pkg/pillar/cmd/zedagent/parsecpuplacement.go @@ -0,0 +1,101 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package zedagent + +import ( + zconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +// parseCPUPlacementPolicy translates the controller's CPU placement intent into +// the device-internal vocabulary. Unknown enum values from a newer controller +// map to "unspecified" and are therefore treated as no preference rather than +// being rejected. +func parseCPUPlacementPolicy(fr *zconfig.VmConfig) types.CPUPlacementPolicy { + if fr == nil { + return types.CPUPlacementPolicy{} + } + return types.CPUPlacementPolicy{ + Policy: parseCPUPolicy(fr.GetCpuPolicy()), + FullPCPUsOnly: fr.GetFullPcpusOnly(), + ThreadsPerCore: fr.GetThreadsPerCore(), + NUMAPolicy: parseCPUNUMAPolicy(fr.GetNumaPolicy()), + IOPlacement: parseCPUIOPlacement(fr.GetIoPlacement()), + IsolationTier: parseCPUIsolationTier(fr.GetIsolationTier()), + DisruptionPolicy: parseCPUDisruptionPolicy(fr.GetDisruptionPolicy()), + } +} + +// cpusPinnedFromPolicy derives the legacy CPUsPinned flag. A policy, when the +// controller sends one, is authoritative; otherwise the legacy pin_cpu flag +// decides, so an older controller keeps behaving exactly as before. +// +// This is also what makes a dedicated policy self-sufficient: the workload no +// longer has to set pin_cpu as well for its CPUs to actually be pinned. +func cpusPinnedFromPolicy(p types.CPUPlacementPolicy, legacyPinCPU bool) bool { + switch p.Policy { + case types.CPUPolicyDedicated: + return true + case types.CPUPolicyShared: + return false + default: + return legacyPinCPU + } +} + +func parseCPUPolicy(p zconfig.CpuPolicy) types.CPUPolicy { + switch p { + case zconfig.CpuPolicy_CPU_POLICY_SHARED: + return types.CPUPolicyShared + case zconfig.CpuPolicy_CPU_POLICY_DEDICATED: + return types.CPUPolicyDedicated + } + return types.CPUPolicyUnspecified +} + +func parseCPUNUMAPolicy(p zconfig.NumaPolicy) types.CPUNUMAPolicy { + switch p { + case zconfig.NumaPolicy_NUMA_POLICY_NONE: + return types.CPUNUMAPolicyNone + case zconfig.NumaPolicy_NUMA_POLICY_BEST_EFFORT: + return types.CPUNUMAPolicyBestEffort + case zconfig.NumaPolicy_NUMA_POLICY_RESTRICTED: + return types.CPUNUMAPolicyRestricted + case zconfig.NumaPolicy_NUMA_POLICY_SINGLE_NUMA_NODE: + return types.CPUNUMAPolicySingleNode + } + return types.CPUNUMAPolicyUnspecified +} + +func parseCPUIOPlacement(p zconfig.IoPlacement) types.CPUIOPlacement { + switch p { + case zconfig.IoPlacement_IO_PLACEMENT_DEDICATED: + return types.CPUIOPlacementDedicated + case zconfig.IoPlacement_IO_PLACEMENT_HOUSEKEEPING: + return types.CPUIOPlacementHousekeeping + } + return types.CPUIOPlacementUnspecified +} + +func parseCPUIsolationTier(t zconfig.IsolationTier) types.CPUIsolationTier { + switch t { + case zconfig.IsolationTier_ISOLATION_TIER_NONE: + return types.CPUIsolationTierNone + case zconfig.IsolationTier_ISOLATION_TIER_SOFT: + return types.CPUIsolationTierSoft + case zconfig.IsolationTier_ISOLATION_TIER_HARD: + return types.CPUIsolationTierHard + } + return types.CPUIsolationTierUnspecified +} + +func parseCPUDisruptionPolicy(p zconfig.DisruptionPolicy) types.CPUDisruptionPolicy { + switch p { + case zconfig.DisruptionPolicy_DISRUPTION_POLICY_ALLOW: + return types.CPUDisruptionPolicyAllow + case zconfig.DisruptionPolicy_DISRUPTION_POLICY_PROTECT: + return types.CPUDisruptionPolicyProtect + } + return types.CPUDisruptionPolicyUnspecified +} diff --git a/pkg/pillar/cmd/zedagent/parsecpuplacement_test.go b/pkg/pillar/cmd/zedagent/parsecpuplacement_test.go new file mode 100644 index 00000000000..fd4c49aa53f --- /dev/null +++ b/pkg/pillar/cmd/zedagent/parsecpuplacement_test.go @@ -0,0 +1,248 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package zedagent + +import ( + "testing" + + zconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +func TestParseCPUPlacementPolicy(t *testing.T) { + tests := []struct { + name string + in *zconfig.VmConfig + want types.CPUPlacementPolicy + }{ + { + name: "nil config", + in: nil, + want: types.CPUPlacementPolicy{}, + }, + { + name: "no policy sent leaves the zero value", + in: &zconfig.VmConfig{}, + want: types.CPUPlacementPolicy{}, + }, + { + name: "whole-core-smt throughput preset", + in: &zconfig.VmConfig{ + CpuPolicy: zconfig.CpuPolicy_CPU_POLICY_DEDICATED, + FullPcpusOnly: true, + ThreadsPerCore: 2, + NumaPolicy: zconfig.NumaPolicy_NUMA_POLICY_SINGLE_NUMA_NODE, + IoPlacement: zconfig.IoPlacement_IO_PLACEMENT_HOUSEKEEPING, + IsolationTier: zconfig.IsolationTier_ISOLATION_TIER_SOFT, + }, + want: types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, + FullPCPUsOnly: true, + ThreadsPerCore: 2, + NUMAPolicy: types.CPUNUMAPolicySingleNode, + IOPlacement: types.CPUIOPlacementHousekeeping, + IsolationTier: types.CPUIsolationTierSoft, + }, + }, + { + name: "one-per-core with protection", + in: &zconfig.VmConfig{ + CpuPolicy: zconfig.CpuPolicy_CPU_POLICY_DEDICATED, + FullPcpusOnly: true, + ThreadsPerCore: 1, + NumaPolicy: zconfig.NumaPolicy_NUMA_POLICY_BEST_EFFORT, + DisruptionPolicy: zconfig.DisruptionPolicy_DISRUPTION_POLICY_PROTECT, + }, + want: types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, + FullPCPUsOnly: true, + ThreadsPerCore: 1, + NUMAPolicy: types.CPUNUMAPolicyBestEffort, + DisruptionPolicy: types.CPUDisruptionPolicyProtect, + }, + }, + { + name: "explicitly shared", + in: &zconfig.VmConfig{CpuPolicy: zconfig.CpuPolicy_CPU_POLICY_SHARED}, + want: types.CPUPlacementPolicy{Policy: types.CPUPolicyShared}, + }, + { + // A newer controller may send values this EVE does not know; they + // must degrade to "no preference", never be misread as a real one. + name: "unknown enum values degrade to unspecified", + in: &zconfig.VmConfig{ + CpuPolicy: zconfig.CpuPolicy(99), + NumaPolicy: zconfig.NumaPolicy(99), + IoPlacement: zconfig.IoPlacement(99), + IsolationTier: zconfig.IsolationTier(99), + DisruptionPolicy: zconfig.DisruptionPolicy(99), + }, + want: types.CPUPlacementPolicy{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseCPUPlacementPolicy(tt.in) + if got != tt.want { + t.Errorf("parseCPUPlacementPolicy()\n got: %+v\nwant: %+v", got, tt.want) + } + }) + } +} + +// The legacy pin_cpu flag must keep working unchanged for controllers that do +// not send a policy, and a policy must win when both are present. +func TestCPUsPinnedFromPolicy(t *testing.T) { + tests := []struct { + name string + policy types.CPUPlacementPolicy + pinCPU bool + wantPinned bool + }{ + {"no policy, pin_cpu false", types.CPUPlacementPolicy{}, false, false}, + {"no policy, pin_cpu true (legacy)", types.CPUPlacementPolicy{}, true, true}, + { + "dedicated policy pins without pin_cpu", + types.CPUPlacementPolicy{Policy: types.CPUPolicyDedicated}, + false, + true, + }, + { + "shared policy overrides a stale pin_cpu", + types.CPUPlacementPolicy{Policy: types.CPUPolicyShared}, + true, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := cpusPinnedFromPolicy(tt.policy, tt.pinCPU); got != tt.wantPinned { + t.Errorf("cpusPinnedFromPolicy() = %v, want %v", got, tt.wantPinned) + } + }) + } +} + +// The tests below pin every enumerator of every placement enum the controller +// can send. They are written against the proto enum's own name map rather than a +// hand-written list of values, so an enumerator added to eve-api fails here +// instead of silently degrading to "unspecified" -- which for a request the +// device cannot honor (ISOLATION_TIER_HARD is the case that matters) would turn +// a fail-closed refusal into a silent downgrade. + +func TestParseCPUPlacementPolicy_CoversEveryCPUPolicy(t *testing.T) { + want := map[zconfig.CpuPolicy]types.CPUPolicy{ + zconfig.CpuPolicy_CPU_POLICY_UNSPECIFIED: types.CPUPolicyUnspecified, + zconfig.CpuPolicy_CPU_POLICY_SHARED: types.CPUPolicyShared, + zconfig.CpuPolicy_CPU_POLICY_DEDICATED: types.CPUPolicyDedicated, + } + for value, name := range zconfig.CpuPolicy_name { + policy := zconfig.CpuPolicy(value) + expected, covered := want[policy] + if !covered { + t.Errorf("%s is not covered by this test, so nothing proves "+ + "parseCPUPolicy maps it", name) + continue + } + got := parseCPUPlacementPolicy(&zconfig.VmConfig{CpuPolicy: policy}).Policy + if got != expected { + t.Errorf("%s parsed as %v, want %v", name, got, expected) + } + } +} + +func TestParseCPUPlacementPolicy_CoversEveryNUMAPolicy(t *testing.T) { + want := map[zconfig.NumaPolicy]types.CPUNUMAPolicy{ + zconfig.NumaPolicy_NUMA_POLICY_UNSPECIFIED: types.CPUNUMAPolicyUnspecified, + zconfig.NumaPolicy_NUMA_POLICY_NONE: types.CPUNUMAPolicyNone, + zconfig.NumaPolicy_NUMA_POLICY_BEST_EFFORT: types.CPUNUMAPolicyBestEffort, + zconfig.NumaPolicy_NUMA_POLICY_RESTRICTED: types.CPUNUMAPolicyRestricted, + zconfig.NumaPolicy_NUMA_POLICY_SINGLE_NUMA_NODE: types.CPUNUMAPolicySingleNode, + } + for value, name := range zconfig.NumaPolicy_name { + policy := zconfig.NumaPolicy(value) + expected, covered := want[policy] + if !covered { + t.Errorf("%s is not covered by this test, so nothing proves "+ + "parseCPUNUMAPolicy maps it", name) + continue + } + got := parseCPUPlacementPolicy(&zconfig.VmConfig{NumaPolicy: policy}).NUMAPolicy + if got != expected { + t.Errorf("%s parsed as %v, want %v", name, got, expected) + } + } +} + +func TestParseCPUPlacementPolicy_CoversEveryIOPlacement(t *testing.T) { + want := map[zconfig.IoPlacement]types.CPUIOPlacement{ + zconfig.IoPlacement_IO_PLACEMENT_UNSPECIFIED: types.CPUIOPlacementUnspecified, + zconfig.IoPlacement_IO_PLACEMENT_DEDICATED: types.CPUIOPlacementDedicated, + zconfig.IoPlacement_IO_PLACEMENT_HOUSEKEEPING: types.CPUIOPlacementHousekeeping, + } + for value, name := range zconfig.IoPlacement_name { + placement := zconfig.IoPlacement(value) + expected, covered := want[placement] + if !covered { + t.Errorf("%s is not covered by this test, so nothing proves "+ + "parseCPUIOPlacement maps it", name) + continue + } + got := parseCPUPlacementPolicy(&zconfig.VmConfig{IoPlacement: placement}).IOPlacement + if got != expected { + t.Errorf("%s parsed as %v, want %v", name, got, expected) + } + } +} + +func TestParseCPUPlacementPolicy_CoversEveryIsolationTier(t *testing.T) { + want := map[zconfig.IsolationTier]types.CPUIsolationTier{ + zconfig.IsolationTier_ISOLATION_TIER_UNSPECIFIED: types.CPUIsolationTierUnspecified, + zconfig.IsolationTier_ISOLATION_TIER_NONE: types.CPUIsolationTierNone, + zconfig.IsolationTier_ISOLATION_TIER_SOFT: types.CPUIsolationTierSoft, + zconfig.IsolationTier_ISOLATION_TIER_HARD: types.CPUIsolationTierHard, + } + for value, name := range zconfig.IsolationTier_name { + tier := zconfig.IsolationTier(value) + expected, covered := want[tier] + if !covered { + t.Errorf("%s is not covered by this test, so nothing proves "+ + "parseCPUIsolationTier maps it", name) + continue + } + got := parseCPUPlacementPolicy(&zconfig.VmConfig{IsolationTier: tier}).IsolationTier + if got != expected { + t.Errorf("%s parsed as %v, want %v", name, got, expected) + } + } + // Hard isolation needs a kernel command-line change, so it has to arrive as + // itself: parsed as anything else the request is silently downgraded to soft + // isolation instead of refused. + if types.CPUIsolationTierHard.SupportedBySoftIsolation() { + t.Error("hard isolation must not be satisfiable by soft isolation") + } +} + +func TestParseCPUPlacementPolicy_CoversEveryDisruptionPolicy(t *testing.T) { + want := map[zconfig.DisruptionPolicy]types.CPUDisruptionPolicy{ + zconfig.DisruptionPolicy_DISRUPTION_POLICY_UNSPECIFIED: types.CPUDisruptionPolicyUnspecified, + zconfig.DisruptionPolicy_DISRUPTION_POLICY_ALLOW: types.CPUDisruptionPolicyAllow, + zconfig.DisruptionPolicy_DISRUPTION_POLICY_PROTECT: types.CPUDisruptionPolicyProtect, + } + for value, name := range zconfig.DisruptionPolicy_name { + policy := zconfig.DisruptionPolicy(value) + expected, covered := want[policy] + if !covered { + t.Errorf("%s is not covered by this test, so nothing proves "+ + "parseCPUDisruptionPolicy maps it", name) + continue + } + got := parseCPUPlacementPolicy( + &zconfig.VmConfig{DisruptionPolicy: policy}).DisruptionPolicy + if got != expected { + t.Errorf("%s parsed as %v, want %v", name, got, expected) + } + } +} diff --git a/pkg/pillar/cmd/zedagent/reportinfo.go b/pkg/pillar/cmd/zedagent/reportinfo.go index bb9dd75f0e2..9e60a70f239 100644 --- a/pkg/pillar/cmd/zedagent/reportinfo.go +++ b/pkg/pillar/cmd/zedagent/reportinfo.go @@ -39,6 +39,23 @@ const ( netDumpInfoOKTopic = agentName + "-info-ok" // Topic for zedagent netdumps of failed info msg publications. netDumpInfoFailTopic = agentName + "-info-fail" + + // deviceAPICapability is the EdgeDevConfig feature level this EVE build + // implements, reported as ZInfoDevice.api_capability. + // + // The enum is monotonic and version-like: "a larger number indicates all + // lower numbers are also supported" (eve-api info.proto), so a controller + // gates a feature on api_capability >= and this single scalar + // stands in for every capability below it. Bump it -- to the immediate + // successor, never skipping a value -- only when the EdgeDevConfig feature + // that the new value names is actually parsed and honored by this build, + // because bumping it also claims everything underneath. + // + // CPU_PLACEMENT_POLICY: the VmConfig CPU placement fields are parsed + // (cmd/zedagent/parsecpuplacement.go) and honored (cmd/domainmgr). This + // also claims APP_INSTANCE_NET_INTERFACE_CHANGE, the value directly below + // it, which this build implements. + deviceAPICapability = info.APICapability_API_CAPABILITY_CPU_PLACEMENT_POLICY ) var ( @@ -646,6 +663,7 @@ func PublishDeviceInfoToZedCloud(ctx *zedagentContext, dest destinationBitset) { ReportDeviceInfo.Capabilities = getCapabilities(ctx) ReportDeviceInfo.OptionalCapabilities = getOptionalCapabilities(ctx) + ReportDeviceInfo.CpuPools = getCPUPools(ctx) devState := getDeviceState(ctx) if ctx.devState != devState { @@ -660,7 +678,7 @@ func PublishDeviceInfoToZedCloud(ctx *zedagentContext, dest destinationBitset) { // device returns a runtime error. Similarly, we only support enforced application network // interface order for the KVM hypervisor. If enabled for application deployed under Xen // or Kubevirt hypervisor, EVE returns error and the application will not be started. - ReportDeviceInfo.ApiCapability = info.APICapability_API_CAPABILITY_APP_INSTANCE_NET_INTERFACE_CHANGE + ReportDeviceInfo.ApiCapability = deviceAPICapability // Report if there is a local override of profile if ctx.getconfigCtx.localCmdAgent.GetCurrentProfile() != @@ -1257,6 +1275,62 @@ func getCapabilities(ctx *zedagentContext) *info.Capabilities { } } +// cpuPoolKindToProto maps the device's pool vocabulary onto the wire enum. +// +// A kind with no mapping is reported as "unspecified", which is an unlabelled +// pool holding real CPUs -- the controller cannot tell what those CPUs are for. +// That can only happen if a pool kind was added without extending this mapping, +// so it is logged as an error rather than passed on quietly. +func cpuPoolKindToProto(kind types.CPUPoolKind) info.CPUPoolKind { + switch kind { + case types.CPUPoolKindHousekeeping: + return info.CPUPoolKind_CPU_POOL_KIND_HOUSEKEEPING + case types.CPUPoolKindDedicated: + return info.CPUPoolKind_CPU_POOL_KIND_DEDICATED + case types.CPUPoolKindIsolated: + return info.CPUPoolKind_CPU_POOL_KIND_ISOLATED + } + log.Errorf("cpuPoolKindToProto: no wire enum for CPU pool kind %s (%d); "+ + "reporting it as unspecified", kind, kind) + return info.CPUPoolKind_CPU_POOL_KIND_UNSPECIFIED +} + +// getCPUPools reports how the node's logical CPUs are partitioned and how much +// of each partition is still available, so a controller can answer "will this +// workload fit?" before a deploy and explain a placement failure after one. +// +// Both the CPU sets and the whole-core counts are carried, because a free thread +// on a partially-taken core cannot satisfy a request for whole physical cores: +// a single "free" number would answer one of the two request shapes wrongly. +func getCPUPools(ctx *zedagentContext) []*info.CPUPoolUtilization { + item, err := ctx.subCPUPoolStatus.Get("global") + if err != nil { + // domainmgr publishes this once it has discovered the CPU topology; before + // that there is nothing to report, which is not an error. + log.Functionf("ctx.subCPUPoolStatus.Get failed: %s", err) + return nil + } + poolStatus, ok := item.(types.CPUPoolStatus) + if !ok { + log.Errorf("Unexpected type for CPUPoolStatus: %T", item) + return nil + } + pools := make([]*info.CPUPoolUtilization, 0, len(poolStatus.Pools)) + for _, pool := range poolStatus.Pools { + pools = append(pools, &info.CPUPoolUtilization{ + Kind: cpuPoolKindToProto(pool.Kind), + CpuIds: pool.CPUs, + FreeCpuIds: pool.FreeCPUs, + TotalThreads: pool.TotalThreads, + AllocatedThreads: pool.AllocatedThreads, + FreeThreads: pool.FreeThreads, + TotalCores: pool.TotalCores, + FreeWholeCores: pool.FreeWholeCores, + }) + } + return pools +} + func getOptionalCapabilities(ctx *zedagentContext) *info.OptionalCapabilities { return &info.OptionalCapabilities{ HvTypeKubevirt: ctx.hvTypeKube, diff --git a/pkg/pillar/cmd/zedagent/reportinfo_test.go b/pkg/pillar/cmd/zedagent/reportinfo_test.go new file mode 100644 index 00000000000..8b58a70985c --- /dev/null +++ b/pkg/pillar/cmd/zedagent/reportinfo_test.go @@ -0,0 +1,188 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package zedagent + +import ( + "testing" + + "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/pubsub" + "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/sirupsen/logrus" +) + +// TestDeviceAPICapabilityCoversCPUPlacement guards the capability gate the +// controller uses to decide whether to offer the CPU placement controls at all. +// The enum is monotonic, so a build that parses and honors the placement fields +// must advertise at least CPU_PLACEMENT_POLICY; anything lower makes the whole +// feature invisible to the controller no matter how well the device implements +// it. +func TestDeviceAPICapabilityCoversCPUPlacement(t *testing.T) { + want := info.APICapability_API_CAPABILITY_CPU_PLACEMENT_POLICY + if deviceAPICapability < want { + t.Errorf("api_capability reported as %s (%d), which is below %s (%d): "+ + "a controller gating on api_capability >= %s will never offer the "+ + "CPU placement fields", + deviceAPICapability, deviceAPICapability, want, want, want) + } +} + +// TestDeviceAPICapabilityIsDefined catches a value that is numerically higher +// than intended -- e.g. a hand-written integer -- which would claim EdgeDevConfig +// features this build does not implement. +func TestDeviceAPICapabilityIsDefined(t *testing.T) { + if _, ok := info.APICapability_name[int32(deviceAPICapability)]; !ok { + t.Errorf("api_capability %d is not a value defined by the vendored eve-api", + deviceAPICapability) + } +} + +// newCPUPoolTestContext builds a zedagentContext whose subCPUPoolStatus holds +// the given pool report, which is all getCPUPools reads. A nil status leaves the +// subscription empty, i.e. domainmgr has not published a report yet. +func newCPUPoolTestContext(t *testing.T, status *types.CPUPoolStatus) *zedagentContext { + t.Helper() + logger = logrus.StandardLogger() + log = base.NewSourceLogObject(logger, agentName, 0) + ps := pubsub.New(pubsub.NewMemoryDriver(), logger, log) + + pub, err := ps.NewPublication(pubsub.PublicationOptions{ + AgentName: "domainmgr", + TopicType: types.CPUPoolStatus{}, + }) + if err != nil { + t.Fatalf("NewPublication(CPUPoolStatus): %v", err) + } + if status != nil { + if err := pub.Publish(status.Key(), *status); err != nil { + t.Fatalf("Publish(CPUPoolStatus): %v", err) + } + } + // Persistent makes Activate load the published report through the driver, so + // Get() sees it without pumping the change channel. + sub, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "domainmgr", + MyAgentName: agentName, + TopicImpl: types.CPUPoolStatus{}, + Persistent: true, + }) + if err != nil { + t.Fatalf("NewSubscription(CPUPoolStatus): %v", err) + } + if err := sub.Activate(); err != nil { + t.Fatalf("Activate(CPUPoolStatus): %v", err) + } + return &zedagentContext{subCPUPoolStatus: sub} +} + +// TestGetCPUPools_CopiesEveryFieldToItsOwnProtoField guards the eight hand-copied +// fields of the pool report. A swapped pair -- free for allocated, threads for +// cores -- would still produce a well-formed message, and the controller would +// answer "will this workload fit?" wrongly in the direction that overcommits the +// node. Every value below is distinct so no two fields can be confused. +func TestGetCPUPools_CopiesEveryFieldToItsOwnProtoField(t *testing.T) { + status := types.CPUPoolStatus{ + Pools: []types.CPUPoolUtilization{ + { + Kind: types.CPUPoolKindHousekeeping, + CPUs: []uint32{0, 1, 2, 3}, + FreeCPUs: []uint32{2, 3}, + TotalThreads: 4, + AllocatedThreads: 1, + FreeThreads: 3, + TotalCores: 2, + FreeWholeCores: 1, + }, + { + Kind: types.CPUPoolKindDedicated, + CPUs: []uint32{4, 5, 6, 7, 8, 9}, + FreeCPUs: []uint32{8, 9}, + TotalThreads: 6, + AllocatedThreads: 5, + FreeThreads: 13, + TotalCores: 11, + FreeWholeCores: 7, + }, + }, + } + ctx := newCPUPoolTestContext(t, &status) + + pools := getCPUPools(ctx) + + if len(pools) != len(status.Pools) { + t.Fatalf("reported %d pools, want %d", len(pools), len(status.Pools)) + } + for i, want := range status.Pools { + got := pools[i] + if got.Kind != cpuPoolKindToProto(want.Kind) { + t.Errorf("pool %d kind %v, want %v", i, got.Kind, + cpuPoolKindToProto(want.Kind)) + } + assertUint32Slice(t, "cpu_ids", got.CpuIds, want.CPUs) + assertUint32Slice(t, "free_cpu_ids", got.FreeCpuIds, want.FreeCPUs) + for _, f := range []struct { + name string + got, want uint32 + }{ + {"total_threads", got.TotalThreads, want.TotalThreads}, + {"allocated_threads", got.AllocatedThreads, want.AllocatedThreads}, + {"free_threads", got.FreeThreads, want.FreeThreads}, + {"total_cores", got.TotalCores, want.TotalCores}, + {"free_whole_cores", got.FreeWholeCores, want.FreeWholeCores}, + } { + if f.got != f.want { + t.Errorf("pool %d %s = %d, want %d", i, f.name, f.got, f.want) + } + } + } +} + +func assertUint32Slice(t *testing.T, name string, got, want []uint32) { + t.Helper() + if len(got) != len(want) { + t.Errorf("%s = %v, want %v", name, got, want) + return + } + for i := range want { + if got[i] != want[i] { + t.Errorf("%s = %v, want %v", name, got, want) + return + } + } +} + +// TestGetCPUPools_SilentBeforeDomainmgrPublishes covers the ordering that holds +// on every boot: device info is sent before domainmgr has discovered the CPU +// topology. No pools is the correct report then, and it must not panic. +func TestGetCPUPools_SilentBeforeDomainmgrPublishes(t *testing.T) { + if pools := getCPUPools(newCPUPoolTestContext(t, nil)); pools != nil { + t.Errorf("reported %v with nothing published, want no pools", pools) + } +} + +// TestCPUPoolKindToProto_CoversEveryKind pins the pool vocabulary onto the wire enum. A kind +// reported under the wrong name tells the controller that CPUs it may not touch +// are free, or the reverse. +func TestCPUPoolKindToProto_CoversEveryKind(t *testing.T) { + tests := []struct { + kind types.CPUPoolKind + want info.CPUPoolKind + }{ + {types.CPUPoolKindUnspecified, info.CPUPoolKind_CPU_POOL_KIND_UNSPECIFIED}, + {types.CPUPoolKindHousekeeping, info.CPUPoolKind_CPU_POOL_KIND_HOUSEKEEPING}, + {types.CPUPoolKindDedicated, info.CPUPoolKind_CPU_POOL_KIND_DEDICATED}, + {types.CPUPoolKindIsolated, info.CPUPoolKind_CPU_POOL_KIND_ISOLATED}, + // A kind added to types without a mapping here: reported as unspecified + // (and logged as an error), never as some other pool's kind. + {types.CPUPoolKind(99), info.CPUPoolKind_CPU_POOL_KIND_UNSPECIFIED}, + } + for _, tt := range tests { + t.Run(tt.kind.String(), func(t *testing.T) { + if got := cpuPoolKindToProto(tt.kind); got != tt.want { + t.Errorf("cpuPoolKindToProto(%s) = %v, want %v", tt.kind, got, tt.want) + } + }) + } +} diff --git a/pkg/pillar/cmd/zedagent/zedagent.go b/pkg/pillar/cmd/zedagent/zedagent.go index cef1bfb280c..7ad7cc7a2d9 100644 --- a/pkg/pillar/cmd/zedagent/zedagent.go +++ b/pkg/pillar/cmd/zedagent/zedagent.go @@ -139,6 +139,7 @@ type zedagentContext struct { subDiskMetric pubsub.Subscription subAppDiskMetric pubsub.Subscription subCapabilities pubsub.Subscription + subCPUPoolStatus pubsub.Subscription subAppInstMetaData pubsub.Subscription subWwanMetrics pubsub.Subscription subWwanStatus pubsub.Subscription @@ -1171,6 +1172,9 @@ func mainEventLoop(zedagentCtx *zedagentContext, stillRunning *time.Ticker) { case change := <-zedagentCtx.subCapabilities.MsgChan(): zedagentCtx.subCapabilities.ProcessChange(change) + case change := <-zedagentCtx.subCPUPoolStatus.MsgChan(): + zedagentCtx.subCPUPoolStatus.ProcessChange(change) + case change := <-zedagentCtx.subBaseOsMgrStatus.MsgChan(): zedagentCtx.subBaseOsMgrStatus.ProcessChange(change) @@ -1916,6 +1920,22 @@ func initPostOnboardSubs(zedagentCtx *zedagentContext) { log.Fatal(err) } + // The node CPU pool report: how the logical CPUs are partitioned between + // housekeeping, dedicated and kernel-isolated, and how much of each is left. + // Only domainmgr can compute it, since it owns the CPU allocator. + zedagentCtx.subCPUPoolStatus, err = ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "domainmgr", + MyAgentName: agentName, + TopicImpl: types.CPUPoolStatus{}, + Activate: true, + Ctx: zedagentCtx, + WarningTime: warningTime, + ErrorTime: errorTime, + }) + if err != nil { + log.Fatal(err) + } + zedagentCtx.subBaseOsMgrStatus, err = ps.NewSubscription(pubsub.SubscriptionOptions{ AgentName: "baseosmgr", MyAgentName: agentName, From a368230bf737ac6b7e911ef1ca94a7852f7a9fb8 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:01:11 +0000 Subject: [PATCH 07/15] zedmanager: publish the set of workloads CPU placement must account for CPU placement has to be a function of the configured set of pinned workloads, but domainmgr only ever sees a DomainConfig, and a DomainConfig cannot exist before a workload's volumes are resolved -- it carries the disk list. So during boot, or while images download at different rates, whichever workload was ready first was placed as though it were alone and took cores the full plan would have assigned elsewhere. The same set of workloads landed differently on each boot. zedmanager knows the whole picture much earlier: it holds every AppInstanceConfig, it owns the profile resolution that decides what is meant to run, and it is the component that withholds the DomainConfig in the first place. It now publishes that demand set -- one aggregate object naming every workload intended to run with its CPU intent -- as soon as the config is resolved, with no dependence on volumes. The set is published as a single object rather than one item per workload on purpose. Per-workload items would leave the consumer planning over whatever had arrived so far, which is the same ordering bug on a faster topic. An empty set is published explicitly, so "no pinned workloads" is distinguishable from "zedmanager has not spoken yet". A workload that is configured but not activated is left out: its cores belong to the workloads that do run, exactly as an assigned PCI device returns to the pool when its workload stops. Also fixes a pre-existing bug this work depends on. The start moment of a delayed workload was computed from a base time set only when zedmanager processed a controller-status message, and the app config regularly won that race -- leaving a start moment derived from the zero time, which is always in the past, so the delay was silently dropped and never recomputed. The base time is now established on first use, so a workload created before that message arrives gets the same start moment as one created after. Signed-off-by: Mikhail Malyshev --- pkg/pillar/cmd/zedmanager/cpudemand.go | 63 ++++++ pkg/pillar/cmd/zedmanager/cpudemand_test.go | 203 ++++++++++++++++++ .../zedmanager/cpuplacementquality_test.go | 155 +++++++++++++ pkg/pillar/cmd/zedmanager/handleclusterapp.go | 5 + pkg/pillar/cmd/zedmanager/start_delay_test.go | 78 +++++++ pkg/pillar/cmd/zedmanager/updatestatus.go | 13 ++ pkg/pillar/cmd/zedmanager/zedmanager.go | 77 +++++-- 7 files changed, 582 insertions(+), 12 deletions(-) create mode 100644 pkg/pillar/cmd/zedmanager/cpudemand.go create mode 100644 pkg/pillar/cmd/zedmanager/cpudemand_test.go create mode 100644 pkg/pillar/cmd/zedmanager/cpuplacementquality_test.go create mode 100644 pkg/pillar/cmd/zedmanager/start_delay_test.go diff --git a/pkg/pillar/cmd/zedmanager/cpudemand.go b/pkg/pillar/cmd/zedmanager/cpudemand.go new file mode 100644 index 00000000000..0daee098bf3 --- /dev/null +++ b/pkg/pillar/cmd/zedmanager/cpudemand.go @@ -0,0 +1,63 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package zedmanager + +import ( + "sort" + + "github.com/lf-edge/eve/pkg/pillar/types" +) + +// publishCPUDemandSet tells domainmgr which applications are intended to run, +// and what CPU placement each asked for. +// +// domainmgr plans CPU placement over the whole set at once, so the layout does +// not depend on the order workloads happen to start in. It cannot derive that +// set from DomainConfig: a DomainConfig only exists once the app's volumes are +// resolved and its network is up, so an app whose image is still downloading is +// invisible and the app that finished first gets planned as if it were alone. +// The CPU intent, by contrast, is known the moment the app config arrives. +// +// INVARIANT: the demand set reaches domainmgr before any DomainConfig for the +// same app. It holds because both are published by this agent, and this one is +// published straight from the app config while a DomainConfig has to wait for +// volume resolution -- work that takes at least one more pubsub round trip. +// domainmgr still tolerates a missing set, but only by falling back to the +// order-dependent behaviour this exists to avoid. +// +// Called on anything that changes the set or the effective-activate decision. +// Publishing an identical set is a no-op in pubsub, so callers need not check. +func publishCPUDemandSet(ctx *zedmanagerContext) { + var set types.CPUDemandSet + for key := range ctx.subAppInstanceConfig.GetAll() { + // The local config wins when there is one, exactly as handleModify + // resolves it, so the demand set describes the config that will + // actually become a DomainConfig. + config := lookupAppInstanceConfig(ctx, key, true) + if config == nil { + continue + } + // An app that is configured but not activated must not hold a CPU + // reservation: its cores belong to the workloads that do run. + if !effectiveActivateCombined(*config, ctx) { + continue + } + set.Apps = append(set.Apps, types.AppCPUDemand{ + UUID: config.UUIDandVersion.UUID, + DisplayName: config.DisplayName, + VCpus: config.FixedResources.VCpus, + CPUsPinned: config.FixedResources.CPUsPinned, + CPUPlacement: config.FixedResources.CPUPlacement, + }) + } + // GetAll iterates a map, so without this an unchanged set would be a + // different object on every publication. + sort.Slice(set.Apps, func(i, j int) bool { + return set.Apps[i].UUID.String() < set.Apps[j].UUID.String() + }) + + if err := ctx.pubCPUDemandSet.Publish(set.Key(), set); err != nil { + log.Errorf("publishCPUDemandSet failed: %v", err) + } +} diff --git a/pkg/pillar/cmd/zedmanager/cpudemand_test.go b/pkg/pillar/cmd/zedmanager/cpudemand_test.go new file mode 100644 index 00000000000..5fb8354ff16 --- /dev/null +++ b/pkg/pillar/cmd/zedmanager/cpudemand_test.go @@ -0,0 +1,203 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package zedmanager + +import ( + "encoding/json" + "sort" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/pubsub" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +// newDemandTestContext builds a zedmanagerContext whose subAppInstanceConfig +// holds the given app configs and whose CPUDemandSet publication can be read +// back, which is all publishCPUDemandSet touches. +func newDemandTestContext(t *testing.T, configs ...types.AppInstanceConfig) *zedmanagerContext { + t.Helper() + logger := logrus.StandardLogger() + log = base.NewSourceLogObject(logger, agentName, 0) + ps := pubsub.New(pubsub.NewMemoryDriver(), logger, log) + + configPub, err := ps.NewPublication(pubsub.PublicationOptions{ + AgentName: "zedagent", + TopicType: types.AppInstanceConfig{}, + }) + assert.NoError(t, err) + for _, config := range configs { + assert.NoError(t, configPub.Publish(config.Key(), config)) + } + // Persistent makes Activate populate the subscription synchronously, so + // GetAll reflects the published configs without pumping the change channel. + configSub, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "zedagent", + MyAgentName: agentName, + TopicImpl: types.AppInstanceConfig{}, + Persistent: true, + }) + assert.NoError(t, err) + assert.NoError(t, configSub.Activate()) + + localSub, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: agentName, + MyAgentName: agentName, + TopicImpl: types.AppInstanceConfig{}, + Persistent: true, + }) + assert.NoError(t, err) + assert.NoError(t, localSub.Activate()) + + demandPub, err := ps.NewPublication(pubsub.PublicationOptions{ + AgentName: agentName, + TopicType: types.CPUDemandSet{}, + }) + assert.NoError(t, err) + + return &zedmanagerContext{ + subAppInstanceConfig: configSub, + subLocalAppInstanceConfig: localSub, + pubCPUDemandSet: demandPub, + } +} + +func publishedDemandSet(t *testing.T, ctx *zedmanagerContext) types.CPUDemandSet { + t.Helper() + item, err := ctx.pubCPUDemandSet.Get("global") + assert.NoError(t, err) + set, ok := item.(types.CPUDemandSet) + assert.True(t, ok) + return set +} + +func demandNames(set types.CPUDemandSet) []string { + var names []string + for _, app := range set.Apps { + names = append(names, app.DisplayName) + } + return names +} + +func appConfigForTest(name string, activate bool, vm types.VmConfig) types.AppInstanceConfig { + return types.AppInstanceConfig{ + UUIDandVersion: types.UUIDandVersion{ + UUID: uuid.NewV5(uuid.NamespaceOID, name), Version: "1", + }, + DisplayName: name, + Activate: activate, + FixedResources: vm, + } +} + +// The set is the whole basis for CPU planning, so it must list exactly the apps +// intended to run. An app that is configured but not activated must not appear: +// holding a CPU reservation for it would keep cores away from the workloads +// that do run. +func TestPublishCPUDemandSet_OnlyAppsIntendedToRun(t *testing.T) { + pinned := types.VmConfig{ + VCpus: 2, + CPUsPinned: true, + CPUPlacement: types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }, + } + ctx := newDemandTestContext(t, + appConfigForTest("running", true, pinned), + appConfigForTest("halted", false, pinned), + appConfigForTest("shared", true, types.VmConfig{VCpus: 4}), + ) + + publishCPUDemandSet(ctx) + + set := publishedDemandSet(t, ctx) + assert.ElementsMatch(t, []string{"running", "shared"}, demandNames(set)) + + // The CPU intent must survive the trip: it is the whole point of the topic. + for _, app := range set.Apps { + if app.DisplayName != "running" { + continue + } + assert.Equal(t, 2, app.VCpus) + assert.True(t, app.CPUsPinned) + assert.Equal(t, types.CPUPolicyDedicated, app.CPUPlacement.Policy) + assert.True(t, app.CPUPlacement.FullPCPUsOnly) + } +} + +// An app whose profile does not match the node's current profile is not +// intended to run, even though its config says Activate. +func TestPublishCPUDemandSet_ProfileDecidesMembership(t *testing.T) { + config := appConfigForTest("profiled", true, types.VmConfig{VCpus: 2}) + config.ProfileList = []string{"daytime"} + ctx := newDemandTestContext(t, config) + + ctx.currentProfile = "nighttime" + publishCPUDemandSet(ctx) + assert.Empty(t, publishedDemandSet(t, ctx).Apps, + "an app the current profile excludes must not hold a CPU reservation") + + ctx.currentProfile = "daytime" + publishCPUDemandSet(ctx) + assert.Equal(t, []string{"profiled"}, demandNames(publishedDemandSet(t, ctx))) +} + +// Publishing an empty set is what lets domainmgr tell "no app asked for CPUs" +// from "zedmanager has not spoken yet", so it must happen even when there is +// nothing to say. +func TestPublishCPUDemandSet_EmptySetIsPublished(t *testing.T) { + ctx := newDemandTestContext(t) + + publishCPUDemandSet(ctx) + + set := publishedDemandSet(t, ctx) + assert.Empty(t, set.Apps) +} + +// TestPublishCPUDemandSet_SortedByUUID guards the sort. subAppInstanceConfig.GetAll +// iterates a map, so without it the same set of apps is published in a different +// order every time: domainmgr sees a changed object, re-plans the placement and +// republishes the pool report on every publication, for no reason at all. +func TestPublishCPUDemandSet_SortedByUUID(t *testing.T) { + // Names, not UUIDs, are what the caller controls; assert below that their + // UUID order really differs from the order they are added in, otherwise this + // test would pass without any sort at all. + insertion := []types.AppInstanceConfig{ + appConfigForTest("alpha", true, types.VmConfig{VCpus: 2}), + appConfigForTest("bravo", true, types.VmConfig{VCpus: 2}), + appConfigForTest("charlie", true, types.VmConfig{VCpus: 2}), + } + var insertionUUIDs []string + for _, config := range insertion { + insertionUUIDs = append(insertionUUIDs, config.UUIDandVersion.UUID.String()) + } + wantUUIDs := append([]string(nil), insertionUUIDs...) + sort.Strings(wantUUIDs) + assert.NotEqual(t, insertionUUIDs, wantUUIDs, + "pick app names whose UUID order differs from the order they are added in") + + ctx := newDemandTestContext(t, insertion...) + + publishCPUDemandSet(ctx) + first := publishedDemandSet(t, ctx) + + var gotUUIDs []string + for _, app := range first.Apps { + gotUUIDs = append(gotUUIDs, app.UUID.String()) + } + assert.Equal(t, wantUUIDs, gotUUIDs) + + // The same set must be byte-identical on the next publication, so an + // unchanged demand set never looks like a change to domainmgr. + publishCPUDemandSet(ctx) + second := publishedDemandSet(t, ctx) + firstJSON, err := json.Marshal(first) + assert.NoError(t, err) + secondJSON, err := json.Marshal(second) + assert.NoError(t, err) + assert.Equal(t, string(firstJSON), string(secondJSON)) +} diff --git a/pkg/pillar/cmd/zedmanager/cpuplacementquality_test.go b/pkg/pillar/cmd/zedmanager/cpuplacementquality_test.go new file mode 100644 index 00000000000..5f22c2cb34e --- /dev/null +++ b/pkg/pillar/cmd/zedmanager/cpuplacementquality_test.go @@ -0,0 +1,155 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package zedmanager + +import ( + "testing" + + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/pubsub" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +// newPlacementQualityTestContext builds a zedmanagerContext wired for the +// activate/inactivate paths: zedmanager's own DomainConfig and AppNetworkConfig +// publications, plus the domainmgr and zedrouter status subscriptions those paths +// read. A nil ds leaves subDomainStatus empty, which is what a torn-down app +// looks like. +func newPlacementQualityTestContext(t *testing.T, ds *types.DomainStatus, + ns *types.AppNetworkStatus) *zedmanagerContext { + t.Helper() + logger := logrus.StandardLogger() + log = base.NewSourceLogObject(logger, agentName, 0) + ps := pubsub.New(pubsub.NewMemoryDriver(), logger, log) + + newPub := func(agent string, topic interface{}) pubsub.Publication { + pub, err := ps.NewPublication(pubsub.PublicationOptions{ + AgentName: agent, + TopicType: topic, + }) + assert.NoError(t, err) + return pub + } + // Persistent makes Activate load the published status through the driver, so + // Get() sees it without pumping the change channel. + newSub := func(agent string, topic interface{}) pubsub.Subscription { + sub, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: agent, + MyAgentName: agentName, + TopicImpl: topic, + Persistent: true, + }) + assert.NoError(t, err) + assert.NoError(t, sub.Activate()) + return sub + } + + domainmgrPub := newPub("domainmgr", types.DomainStatus{}) + if ds != nil { + assert.NoError(t, domainmgrPub.Publish(ds.Key(), *ds)) + } + zedrouterPub := newPub("zedrouter", types.AppNetworkStatus{}) + if ns != nil { + assert.NoError(t, zedrouterPub.Publish(ns.Key(), *ns)) + } + + return &zedmanagerContext{ + pubDomainConfig: newPub(agentName, types.DomainConfig{}), + pubAppNetworkConfig: newPub(agentName, types.AppNetworkConfig{}), + subDomainStatus: newSub("domainmgr", types.DomainStatus{}), + subAppNetworkStatus: newSub("zedrouter", types.AppNetworkStatus{}), + } +} + +// placementQualityFixture returns a running app and the domainmgr/zedrouter +// statuses that let doActivate run to the end. +func placementQualityFixture(quality types.CPUPlacementQuality) (types.AppInstanceConfig, + *types.AppInstanceStatus, *types.DomainStatus, *types.AppNetworkStatus) { + + uuidAndVersion := types.UUIDandVersion{ + UUID: uuid.NewV5(uuid.NamespaceOID, "pinned-app"), + Version: "1", + } + config := types.AppInstanceConfig{ + UUIDandVersion: uuidAndVersion, + DisplayName: "pinned-app", + Activate: true, + FixedResources: types.VmConfig{VCpus: 4, Memory: 1024, CPUsPinned: true}, + } + // Activated skips the memory admission check, which is the state an app is in + // when domainmgr re-evaluates its placement. + status := &types.AppInstanceStatus{ + UUIDandVersion: uuidAndVersion, + DisplayName: "pinned-app", + State: types.RUNNING, + Activated: true, + } + ds := &types.DomainStatus{ + UUIDandVersion: uuidAndVersion, + DisplayName: "pinned-app", + DomainName: "pinned-app.1.1", + State: types.RUNNING, + Activated: true, + PlacementQuality: quality, + } + ns := &types.AppNetworkStatus{ + UUIDandVersion: uuidAndVersion, + DisplayName: "pinned-app", + Activated: true, + } + return config, status, ds, ns +} + +// TestDoActivate_CopiesPlacementQualityFromDomainStatus is the zedmanager link in +// the chain that carries a sub-optimal placement to the controller: domainmgr +// computes it on DomainStatus, zedmanager copies it onto AppInstanceStatus, and +// zedagent turns it into an advisory. Without this copy the device works out that +// a repack would help and then keeps it to itself. +func TestDoActivate_CopiesPlacementQualityFromDomainStatus(t *testing.T) { + config, status, ds, ns := placementQualityFixture(types.CPUPlacementQualityNeedsRepack) + ctx := newPlacementQualityTestContext(t, ds, ns) + + doActivate(ctx, status.Key(), config, status) + + assert.Equal(t, types.CPUPlacementQualityNeedsRepack, status.PlacementQuality) + // It is status, not a failure: a workload placed sub-optimally keeps running. + assert.False(t, status.HasError()) + assert.Equal(t, types.RUNNING, status.State) +} + +// TestDoActivate_FollowsPlacementQualityBackToOptimal covers the other direction: +// once neighbouring workloads have moved, the same app is optimally placed and +// the advisory has to go away. A copy that only ever set "needs repack" would +// leave the controller nagging about a workload that is now placed as well as it +// can be. +func TestDoActivate_FollowsPlacementQualityBackToOptimal(t *testing.T) { + config, status, ds, ns := placementQualityFixture(types.CPUPlacementQualityOptimal) + status.PlacementQuality = types.CPUPlacementQualityNeedsRepack + ctx := newPlacementQualityTestContext(t, ds, ns) + + doActivate(ctx, status.Key(), config, status) + + assert.Equal(t, types.CPUPlacementQualityOptimal, status.PlacementQuality) +} + +// TestDoInactivate_ClearsPlacementQuality checks the teardown side: a torn-down +// workload holds no CPUs, so there is no placement left to judge. A stale +// "needs repack" would keep an advisory on the wire for an app that is not even +// running, and would survive into the next activation as a wrong starting value. +func TestDoInactivate_ClearsPlacementQuality(t *testing.T) { + _, status, _, _ := placementQualityFixture(types.CPUPlacementQualityNeedsRepack) + status.PlacementQuality = types.CPUPlacementQualityNeedsRepack + // Nothing published for the app any more: domainmgr has removed the domain + // and zedrouter the network. + ctx := newPlacementQualityTestContext(t, nil, nil) + + changed, done := doInactivate(ctx, status.UUIDandVersion.UUID, status) + + assert.True(t, done) + assert.True(t, changed) + assert.Equal(t, types.CPUPlacementQualityUnspecified, status.PlacementQuality) +} diff --git a/pkg/pillar/cmd/zedmanager/handleclusterapp.go b/pkg/pillar/cmd/zedmanager/handleclusterapp.go index 7337e9a04a4..23b984e1271 100644 --- a/pkg/pillar/cmd/zedmanager/handleclusterapp.go +++ b/pkg/pillar/cmd/zedmanager/handleclusterapp.go @@ -31,6 +31,11 @@ func handleENClusterAppStatusImpl(ctx *zedmanagerContext, key string, status *ty aiStatus := lookupAppInstanceStatus(ctx, key) log.Noticef("handleENClusterAppStatusImpl(%s) for app-status %v aiStatus %v", key, status, aiStatus) + // In cluster mode the effective-activate decision also depends on where + // kubernetes scheduled the app, so this changes which apps are intended to + // run here without any config change. + publishCPUDemandSet(ctx) + if status.ScheduledOnThisNode { if aiStatus == nil { // This could happen if app failover to other node and failing back to this designated node. diff --git a/pkg/pillar/cmd/zedmanager/start_delay_test.go b/pkg/pillar/cmd/zedmanager/start_delay_test.go new file mode 100644 index 00000000000..4da25169d09 --- /dev/null +++ b/pkg/pillar/cmd/zedmanager/start_delay_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package zedmanager + +import ( + "testing" + "time" + + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +// newStartDelayTestContext builds the minimal context the start delay +// computation needs: no base time yet and a stopped fallback timer. +func newStartDelayTestContext(t *testing.T) *zedmanagerContext { + t.Helper() + log = base.NewSourceLogObject(logrus.StandardLogger(), agentName, 0) + + timer := time.NewTimer(waitForAppsToStartTimeout) + if !timer.Stop() { + <-timer.C + } + return &zedmanagerContext{priorityStartTimer: timer} +} + +// TestAppStartTimeConfigBeforeZedAgentStatus covers the ordering where an +// AppInstanceConfig is handled before the ZedAgentStatus that reports a +// successful config get. The start moment must still be a real one, otherwise +// the delay is added to the zero time and the app is never held back. +func TestAppStartTimeConfigBeforeZedAgentStatus(t *testing.T) { + ctx := newStartDelayTestContext(t) + const delay = 3 * time.Minute + config := types.AppInstanceConfig{Delay: delay} + + before := time.Now() + first := appStartTime(ctx, config) + after := time.Now() + + assert.False(t, first.IsZero()) + assert.False(t, first.Before(before.Add(delay))) + assert.False(t, first.After(after.Add(delay))) + // The whole point of the delay: the app is not allowed to start yet. + assert.True(t, time.Now().Before(first)) + + // The status that normally establishes the base time arrives afterwards. It + // must not move the start moment of an app created before it, so that the + // two orderings are indistinguishable. + handleZedAgentStatusImpl(ctx, "zedagent", + types.ZedAgentStatus{ConfigGetStatus: types.ConfigGetSuccess}) + assert.Equal(t, first, appStartTime(ctx, config)) + + // An app without a delay is not held back. + assert.False(t, time.Now().Before(appStartTime(ctx, types.AppInstanceConfig{}))) + + // Establishing the base time must arm the fallback that releases + // low-priority apps; Stop reports true only for a timer that is running. + assert.True(t, ctx.priorityStartTimer.Stop()) +} + +// TestAppStartTimeZedAgentStatusFirst is the ordering that used to work: the +// base time is known before any app config arrives. +func TestAppStartTimeZedAgentStatusFirst(t *testing.T) { + ctx := newStartDelayTestContext(t) + const delay = 3 * time.Minute + + handleZedAgentStatusImpl(ctx, "zedagent", + types.ZedAgentStatus{ConfigGetStatus: types.ConfigGetSuccess}) + baseTime := ctx.delayBaseTime + assert.False(t, baseTime.IsZero()) + + assert.Equal(t, baseTime.Add(delay), + appStartTime(ctx, types.AppInstanceConfig{Delay: delay})) + // A later app config must not re-base the delay. + assert.Equal(t, baseTime, ctx.delayBaseTime) +} diff --git a/pkg/pillar/cmd/zedmanager/updatestatus.go b/pkg/pillar/cmd/zedmanager/updatestatus.go index 504ba044e3d..64f2858d959 100644 --- a/pkg/pillar/cmd/zedmanager/updatestatus.go +++ b/pkg/pillar/cmd/zedmanager/updatestatus.go @@ -880,6 +880,13 @@ func doActivate(ctx *zedmanagerContext, uuidStr string, if c { changed = true } + // How good the CPU placement domainmgr found is. Advisory: it travels as a + // plain status field rather than an error so that a workload which is merely + // placed sub-optimally is never mistaken for a failed one. + if status.PlacementQuality != ds.PlacementQuality { + status.PlacementQuality = ds.PlacementQuality + changed = true + } // Are we doing a restart? if status.RestartInprogress == types.BringDown { if dc.Activate { @@ -1154,6 +1161,12 @@ func doInactivate(ctx *zedmanagerContext, appInstID uuid.UUID, } log.Functionf("Done with DomainStatus removal/deactivate for %s", uuidStr) + // The workload holds no CPUs any more, so there is no placement to judge. + if status.PlacementQuality != types.CPUPlacementQualityUnspecified { + status.PlacementQuality = types.CPUPlacementQualityUnspecified + changed = true + } + if uninstall { if lookupAppNetworkConfig(ctx, uuidStr) != nil { unpublishAppNetworkConfig(ctx, uuidStr) diff --git a/pkg/pillar/cmd/zedmanager/zedmanager.go b/pkg/pillar/cmd/zedmanager/zedmanager.go index 0b57db3000b..749211979d3 100644 --- a/pkg/pillar/cmd/zedmanager/zedmanager.go +++ b/pkg/pillar/cmd/zedmanager/zedmanager.go @@ -54,6 +54,7 @@ type zedmanagerContext struct { pubAppNetworkConfig pubsub.Publication subAppNetworkStatus pubsub.Subscription pubDomainConfig pubsub.Publication + pubCPUDemandSet pubsub.Publication subDomainStatus pubsub.Subscription subENClusterAppStatus pubsub.Subscription subGlobalConfig pubsub.Subscription @@ -181,6 +182,15 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar ctx.pubDomainConfig = pubDomainConfig pubDomainConfig.ClearRestarted() + pubCPUDemandSet, err := ps.NewPublication(pubsub.PublicationOptions{ + AgentName: agentName, + TopicType: types.CPUDemandSet{}, + }) + if err != nil { + log.Fatal(err) + } + ctx.pubCPUDemandSet = pubCPUDemandSet + // Persist purge counter for each application. mapPublisher, err := objtonum.NewObjNumPublisher( log, ps, agentName, true, &types.UuidToNum{}) @@ -460,7 +470,7 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar // Low-priority apps are released by an event: either a high-priority app // reaching a running state (observed via subAppInstanceStatus below) or the // priorityStartTimer expiring. Create it stopped; it is armed when - // delayBaseTime is first set in handleZedAgentStatusImpl. + // delayBaseTime is first set, see ensureDelayBaseTime. ctx.priorityStartTimer = time.NewTimer(waitForAppsToStartTimeout) if !ctx.priorityStartTimer.Stop() { <-ctx.priorityStartTimer.C @@ -740,6 +750,11 @@ func handleAADelete(ctxArg interface{}, key string, statusArg interface{}) { func handleConfigRestart(ctxArg interface{}, restartCounter int) { ctx := ctxArg.(*zedmanagerContext) log.Functionf("handleConfigRestart(%d)", restartCounter) + // The initial config has been delivered, so the demand set is now complete + // -- possibly empty. Publish it either way: domainmgr must be able to tell + // "no app asked for CPUs" from "zedmanager has not spoken yet", and only an + // explicit publication says the former. + publishCPUDemandSet(ctx) if restartCounter != 0 { ctx.pubAppNetworkConfig.SignalRestarted() } @@ -890,6 +905,11 @@ func handleAppInstanceConfigDelete(ctxArg interface{}, key string, log.Functionf("handleAppInstanceConfigDelete(%s)", key) ctx := ctxArg.(*zedmanagerContext) + // pubsub drops the key before calling us, so the set built here no longer + // holds the deleted app -- its CPUs are free for the plan again. Done even + // when there is no status, so a config that never reached a status does not + // leave a reservation behind. + publishCPUDemandSet(ctx) status := lookupAppInstanceStatus(ctx, key) if status == nil { log.Functionf("handleAppInstanceConfigDelete: unknown %s", key) @@ -1214,6 +1234,36 @@ func serializeAppInstanceConfigToSnapshot(config types.AppInstanceConfig, snapsh return nil } +// ensureDelayBaseTime establishes the moment from which the configured +// application delays are counted, unless it is already known. +func ensureDelayBaseTime(ctx *zedmanagerContext) { + if !ctx.delayBaseTime.IsZero() { + return + } + ctx.delayBaseTime = time.Now() + // Arm the fallback that releases low-priority apps once the startup + // window closes. The extra margin keeps the timer from firing a hair + // before highPriorityAppsPending sees the deadline as passed, which + // would leave apps held with no further wakeup. Guarded because + // subAppInstanceConfig is activated before the timer is created. + if ctx.priorityStartTimer != nil { + ctx.priorityStartTimer.Reset(waitForAppsToStartTimeout + time.Second) + } +} + +// appStartTime returns the moment at which the application is allowed to start, +// i.e. the base time plus the configured delay. +// +// The base time is established here as well as on ZedAgentStatus, because an +// AppInstanceConfig can reach zedmanager first: the two travel over separate +// pubsub channels and an app config arriving is by itself proof that the device +// got its configuration. Otherwise the delay would be added to the zero time, +// putting the start moment in year 1 and starting the app immediately. +func appStartTime(ctx *zedmanagerContext, config types.AppInstanceConfig) time.Time { + ensureDelayBaseTime(ctx) + return ctx.delayBaseTime.Add(config.Delay) +} + func handleCreate(ctxArg interface{}, key string, configArg interface{}) { ctx := ctxArg.(*zedmanagerContext) @@ -1222,6 +1272,9 @@ func handleCreate(ctxArg interface{}, key string, log.Functionf("handleCreate(%v) for %s", config.UUIDandVersion, config.DisplayName) + // Before doUpdate, so domainmgr knows this app is coming before it is ever + // asked to place another one. + publishCPUDemandSet(ctx) handleCreateAppInstanceStatus(ctx, config) } @@ -1237,8 +1290,7 @@ func handleCreateAppInstanceStatus(ctx *zedmanagerContext, config types.AppInsta IsDesignatedNodeID: config.IsDesignatedNodeID, } - // Calculate the moment when the application should start, taking into account the configured delay - status.StartTime = ctx.delayBaseTime.Add(config.Delay) + status.StartTime = appStartTime(ctx, config) restoreAvailableSnapshots(&status) @@ -1344,6 +1396,10 @@ func handleModify(ctxArg interface{}, key string, config = *localConfig } + // A modify can change vCPU count, the CPU policy, or whether the app is + // activated at all -- all of which move the demand set. + publishCPUDemandSet(ctx) + // Check if we need to roll back to a snapshot if config.Snapshot.RollbackCmd.Counter > oldConfig.Snapshot.RollbackCmd.Counter { log.Noticef("handleModify(%v) for %s: Snapshot to be rolled back: %v", @@ -1368,7 +1424,7 @@ func handleModify(ctxArg interface{}, key string, return } - status.StartTime = ctx.delayBaseTime.Add(config.Delay) + status.StartTime = appStartTime(ctx, config) updateSnapshotsInAIStatus(status, config) @@ -1782,14 +1838,7 @@ func handleZedAgentStatusImpl(ctxArg interface{}, key string, // When getting the config successfully for the first time (get from the controller or read from the file), consider // the device as ready to start apps. Hence, count the app delay timeout from now. if status.ConfigGetStatus == types.ConfigGetSuccess || status.ConfigGetStatus == types.ConfigGetReadSaved { - if ctxPtr.delayBaseTime.IsZero() { - ctxPtr.delayBaseTime = time.Now() - // Arm the fallback that releases low-priority apps once the - // startup window closes. The extra margin keeps the timer from - // firing a hair before highPriorityAppsPending sees the deadline - // as passed, which would leave apps held with no further wakeup. - ctxPtr.priorityStartTimer.Reset(waitForAppsToStartTimeout + time.Second) - } + ensureDelayBaseTime(ctxPtr) } if ctxPtr.currentProfile != status.CurrentProfile { @@ -1831,6 +1880,10 @@ func handleHostMemoryImpl(ctxArg interface{}, key string, // updateBasedOnProfile check all app instances with ctx.currentProfile and oldProfile // update AppInstance if change in effective activate detected func updateBasedOnProfile(ctx *zedmanagerContext, oldProfile string) { + // A profile change activates and deactivates apps without any config + // change, so it moves the demand set. Republish before acting on it, so an + // app the new profile starts is in the set before its DomainConfig is. + publishCPUDemandSet(ctx) pub := ctx.subAppInstanceConfig items := pub.GetAll() for _, c := range items { From d7df42ce8914e6bcd67b3a9468467599afa6ac25 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:04:25 +0000 Subject: [PATCH 08/15] domainmgr: place pinned workloads as a set, and hold a failure until told Placement now runs over the whole demand set published by zedmanager rather than over whichever DomainConfigs have arrived. The result for a given set of workloads is therefore the same regardless of the order they were configured, started or delayed in, and the same across a reboot -- properties an operator depends on when a workload's performance was validated against a specific placement. The plan is derived, never stored. Persisting an assignment would create a second source of truth that can disagree with the hardware after a CPU is offlined, a NUMA node changes or the config changes, and the failure mode of stale placement data is silent and hard to diagnose. Determinism comes from ordering the batch by how constrained each workload is and breaking ties on the workload's identity, so recomputation reproduces the same answer. A whole-core request consumes every thread of its cores. When only one thread per core is wanted, the sibling is parked -- held by that workload and offered to nobody. This is the point of asking for a whole core: a best-effort workload running on the parked sibling would evict the cache lines and contend for the execution units the request exists to protect. Parked threads are reported as consumed in the pool utilization rather than as spare capacity, so a controller sees the true remaining headroom. Placement failures are terminal. A workload that cannot be placed stops with an error naming the cause -- a shortage, a shortage a repack would fix, or a request nothing could satisfy -- and stays stopped until an operator changes the config, which is how EVE already treats an unavailable PCI device. Retrying would silently place the workload the moment some unrelated workload happened to release cores, at an arbitrary time, with no operator awareness that its performance envelope had changed. Two long-standing behaviours are corrected. The operator-editable override on /persist can now enable pinning for a workload the controller did not pin, not only disable it, which is what makes it usable for on-device diagnosis. And housekeeping IO placement no longer draws its CPUs from a set that could include cores already promised to another workload. Signed-off-by: Mikhail Malyshev --- pkg/pillar/cmd/domainmgr/cpudemand_test.go | 240 +++++ pkg/pillar/cmd/domainmgr/cpuplan.go | 348 +++++++ pkg/pillar/cmd/domainmgr/cpuplan_test.go | 78 ++ pkg/pillar/cmd/domainmgr/cpupools.go | 87 ++ pkg/pillar/cmd/domainmgr/cpupools_test.go | 137 +++ .../cmd/domainmgr/cpuredistribute_test.go | 160 ++++ pkg/pillar/cmd/domainmgr/domainmgr.go | 587 ++++++++++-- pkg/pillar/cmd/domainmgr/pinningconfig.go | 235 +++++ .../cmd/domainmgr/pinningconfig_test.go | 153 ++++ .../cmd/domainmgr/placementfixes_test.go | 286 ++++++ pkg/pillar/cmd/domainmgr/placementpolicy.go | 455 ++++++++++ .../cmd/domainmgr/placementpolicy_test.go | 852 ++++++++++++++++++ 12 files changed, 3546 insertions(+), 72 deletions(-) create mode 100644 pkg/pillar/cmd/domainmgr/cpudemand_test.go create mode 100644 pkg/pillar/cmd/domainmgr/cpuplan.go create mode 100644 pkg/pillar/cmd/domainmgr/cpuplan_test.go create mode 100644 pkg/pillar/cmd/domainmgr/cpupools.go create mode 100644 pkg/pillar/cmd/domainmgr/cpupools_test.go create mode 100644 pkg/pillar/cmd/domainmgr/cpuredistribute_test.go create mode 100644 pkg/pillar/cmd/domainmgr/pinningconfig.go create mode 100644 pkg/pillar/cmd/domainmgr/pinningconfig_test.go create mode 100644 pkg/pillar/cmd/domainmgr/placementfixes_test.go create mode 100644 pkg/pillar/cmd/domainmgr/placementpolicy.go create mode 100644 pkg/pillar/cmd/domainmgr/placementpolicy_test.go diff --git a/pkg/pillar/cmd/domainmgr/cpudemand_test.go b/pkg/pillar/cmd/domainmgr/cpudemand_test.go new file mode 100644 index 00000000000..da147efcc58 --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/cpudemand_test.go @@ -0,0 +1,240 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "reflect" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/pubsub" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +// testCPUDemandSub publishes a demand set as zedmanager would and returns a +// subscription over it, so planning has the whole intended set to work from. +func testCPUDemandSub(t *testing.T, ps *pubsub.PubSub, + apps ...types.AppCPUDemand) pubsub.Subscription { + t.Helper() + // Persistent on both ends so the subscription populates from what is + // already published, the way a real subscriber picks up existing state. + pub, err := ps.NewPublication(pubsub.PublicationOptions{ + AgentName: "zedmanager", + TopicType: types.CPUDemandSet{}, + Persistent: true, + }) + if err != nil { + t.Fatalf("NewPublication(CPUDemandSet): %v", err) + } + set := types.CPUDemandSet{Apps: apps} + if err := pub.Publish(set.Key(), set); err != nil { + t.Fatalf("Publish(CPUDemandSet): %v", err) + } + sub, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "zedmanager", + MyAgentName: agentName, + TopicImpl: types.CPUDemandSet{}, + Activate: true, + Persistent: true, + }) + if err != nil { + t.Fatalf("NewSubscription(CPUDemandSet): %v", err) + } + return sub +} + +// wholeCoreDemand is an app the controller asked to place on whole physical +// cores. +func wholeCoreDemand(name string, vcpus int) types.AppCPUDemand { + return types.AppCPUDemand{ + UUID: uuid.NewV5(uuid.NamespaceOID, name), + DisplayName: name, + VCpus: vcpus, + CPUsPinned: true, + CPUPlacement: types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }, + } +} + +// threadGranularDemand is an app the controller asked to pin at thread +// granularity: dedicated CPUs, but without full_pcpus_only. It takes individual +// threads, so the physical cores it lands on are left half-owned. +func threadGranularDemand(name string, vcpus int) types.AppCPUDemand { + return types.AppCPUDemand{ + UUID: uuid.NewV5(uuid.NamespaceOID, name), + DisplayName: name, + VCpus: vcpus, + CPUsPinned: true, + CPUPlacement: types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, + }, + } +} + +// assignedCPUs flattens a plan into UUID -> occupied CPUs, which is what a +// caller of the plan actually depends on. +func assignedCPUs(plan map[uuid.UUID]cpuallocator.Result) map[uuid.UUID][]uint32 { + out := map[uuid.UUID][]uint32{} + for id, result := range plan { + if result.Status != cpuallocator.Success || result.Assignment == nil { + continue + } + out[id] = assignmentCPUs(result.Assignment) + } + return out +} + +// The bug this replaces: planning ran over the DomainConfigs that had arrived, +// and a DomainConfig only exists once an app's volumes are resolved. Two apps +// whose images download at different speeds therefore got a different layout on +// every boot -- whichever activated first was planned as if it were alone. +// +// Planning must be a function of the demand set alone: the same set, in any +// order, with any subset having reached a DomainConfig, must place identically. +func TestPlanPinnedPlacement_IndependentOfArrivalOrder(t *testing.T) { + isolatePinningOverride(t) + smt := wholeCoreDemand("smt-app", 2) + core := wholeCoreDemand("core-app", 2) + core.CPUPlacement.ThreadsPerCore = 1 + + // The reference: the whole set known, nothing activated yet. + ps := testPubSub(t) + ctx := &domainContext{ + placer: testPlacer(t), + subCPUDemandSet: testCPUDemandSub(t, ps, smt, core), + subDomainConfig: testDomainConfigSub(t, ps), + } + want := assignedCPUs(planPinnedPlacement(ctx)) + if len(want) != 2 { + t.Fatalf("both apps must be planned, got %v", want) + } + + // The same set, entries in the other order, and with only one of the two + // apps far enough along to have a DomainConfig -- the situation that used + // to decide the layout. + dc := pinnedConfigForTest("smt-app", smt.CPUPlacement) + for _, tt := range []struct { + name string + apps []types.AppCPUDemand + configs []types.DomainConfig + }{ + {"reversed demand order", []types.AppCPUDemand{core, smt}, nil}, + {"only one app has a DomainConfig", []types.AppCPUDemand{smt, core}, + []types.DomainConfig{dc}}, + {"reversed, only one DomainConfig", []types.AppCPUDemand{core, smt}, + []types.DomainConfig{dc}}, + } { + t.Run(tt.name, func(t *testing.T) { + ps := testPubSub(t) + ctx := &domainContext{ + placer: testPlacer(t), + subCPUDemandSet: testCPUDemandSub(t, ps, tt.apps...), + subDomainConfig: testDomainConfigSub(t, ps, tt.configs...), + } + if got := assignedCPUs(planPinnedPlacement(ctx)); !reflect.DeepEqual(got, want) { + t.Errorf("placement = %v, want %v -- the plan must depend on the "+ + "demand set alone", got, want) + } + }) + } +} + +// An app that has a DomainConfig but is not in the demand set is on its way out +// (deleted, or deactivated by a profile change). Planning CPUs for it would set +// aside cores nothing is going to use. +func TestPlanPinnedPlacement_DemandSetIsAuthoritative(t *testing.T) { + isolatePinningOverride(t) + ps := testPubSub(t) + stays := wholeCoreDemand("stays", 2) + leaving := pinnedConfigForTest("leaving", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }) + ctx := &domainContext{ + placer: testPlacer(t), + subCPUDemandSet: testCPUDemandSub(t, ps, stays), + subDomainConfig: testDomainConfigSub(t, ps, leaving), + } + + plan := planPinnedPlacement(ctx) + if _, ok := plan[leaving.UUIDandVersion.UUID]; ok { + t.Error("an app absent from the demand set must not be planned for") + } + if _, ok := plan[stays.UUID]; !ok { + t.Errorf("the app in the demand set must be planned, got %v", plan) + } +} + +// The demand set carries the controller's intent only: zedmanager cannot see +// the operator-editable /persist override. domainmgr must therefore still apply +// it, or the override -- the only way to ask for whole-core placement on a +// device whose controller knows nothing about the policy -- would plan nothing +// and the workload would land wherever it fitted on the day. +func TestPlanPinnedPlacement_PersistOverrideStillPins(t *testing.T) { + isolatePinningOverride(t) + ps := testPubSub(t) + // No controller intent at all: not pinned, no policy. + app := types.AppCPUDemand{ + UUID: uuid.NewV5(uuid.NamespaceOID, "override-planned"), + DisplayName: "override-planned", + VCpus: 2, + } + ctx := &domainContext{ + placer: testPlacer(t), + subCPUDemandSet: testCPUDemandSub(t, ps, app), + subDomainConfig: testDomainConfigSub(t, ps), + } + + if plan := planPinnedPlacement(ctx); len(plan) != 0 { + t.Fatalf("without an override there is nothing to plan, got %v", plan) + } + + writePinningEntryForTest(t, app.UUID, &PinningEntry{ + CPUPolicy: "static", PolicyOptions: fullPCPUs(), ThreadsPerCore: 1, + }) + plan := planPinnedPlacement(ctx) + result, ok := plan[app.UUID] + if !ok || result.Status != cpuallocator.Success { + t.Fatalf("the /persist override must reach the plan, got %v", plan) + } + // one-per-core: one host CPU per vCPU, with the SMT sibling parked. + if len(result.Assignment.OrderedHostCPUs) != 2 || len(result.Assignment.ParkedCPUs) != 2 { + t.Errorf("override asked for one-per-core, got cpus=%v parked=%v", + result.Assignment.OrderedHostCPUs, result.Assignment.ParkedCPUs) + } +} + +// "No app asked for CPUs" and "zedmanager has not spoken yet" mean opposite +// things. An empty set is an answer and must be honoured; a missing one must +// fall back rather than leave a workload unplanned. +func TestPlanPinnedPlacement_EmptySetIsNotAMissingSet(t *testing.T) { + isolatePinningOverride(t) + config := pinnedConfigForTest("pinned", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }) + + ps := testPubSub(t) + empty := &domainContext{ + placer: testPlacer(t), + subCPUDemandSet: testCPUDemandSub(t, ps), + subDomainConfig: testDomainConfigSub(t, ps, config), + } + if plan := planPinnedPlacement(empty); len(plan) != 0 { + t.Errorf("an explicitly empty demand set means no app is intended to "+ + "run; nothing may be planned, got %v", plan) + } + + ps = testPubSub(t) + missing := &domainContext{ + placer: testPlacer(t), + subDomainConfig: testDomainConfigSub(t, ps, config), + } + plan := planPinnedPlacement(missing) + if _, ok := plan[config.UUIDandVersion.UUID]; !ok { + t.Errorf("without a demand set the DomainConfigs must still be planned "+ + "-- a missing message may not stop a workload from starting, got %v", + plan) + } +} diff --git a/pkg/pillar/cmd/domainmgr/cpuplan.go b/pkg/pillar/cmd/domainmgr/cpuplan.go new file mode 100644 index 00000000000..e92e78512a1 --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/cpuplan.go @@ -0,0 +1,348 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + + "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/cputopology" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +// cpuPlanFile is where the current placement plan is mirrored. It is derived +// state, recomputed from the app configs and the topology, and exists on disk +// only so an operator can see what the device decided and why a workload is +// where it is. Nothing reads it back. +var cpuPlanFile = filepath.Join(runDirname, "cpuplan.json") + +// plannedPlacement is one workload's entry in the mirrored plan. +type plannedPlacement struct { + UUID string `json:"uuid"` + DisplayName string `json:"display_name"` + Mode string `json:"mode"` + VCPUs int `json:"vcpus"` + Status string `json:"status"` + HostCPUs []uint32 `json:"host_cpus,omitempty"` + ParkedCPUs []uint32 `json:"parked_cpus,omitempty"` + Message string `json:"message,omitempty"` +} + +// cpuPlan is the mirrored plan as a whole. +type cpuPlan struct { + Comment string `json:"_comment"` + Workloads []plannedPlacement `json:"workloads"` +} + +// plannedIntents is the set of workloads placement is planned over: every app +// the controller intends to run on this node, whether or not it has got as far +// as a DomainConfig. +// +// It has to come from zedmanager's demand set rather than from the DomainConfigs +// received so far, because a DomainConfig is only published once the app's +// volumes are resolved and its network is up. Planning over what has arrived +// makes the layout depend on image download order: the app that finished +// downloading first is planned as if it were alone and takes CPUs the full plan +// would have given to another, so the same config lays out differently after a +// reboot. +// +// The fallback exists only so a missing pubsub message can never stop a +// workload from starting. It restores the order-dependent behaviour, which is +// why it is loud. +func plannedIntents(ctx *domainContext) []cpuIntent { + if ctx.subCPUDemandSet != nil { + if item, err := ctx.subCPUDemandSet.Get("global"); err == nil { + set, ok := item.(types.CPUDemandSet) + if ok { + intents := make([]cpuIntent, 0, len(set.Apps)) + for _, app := range set.Apps { + intents = append(intents, intentOfDemand(app)) + } + return intents + } + log.Errorf("plannedIntents: unexpected type %T for CPUDemandSet", item) + } + } + // Not an empty set: an empty one is published explicitly, and arrives as a + // CPUDemandSet with no apps. + log.Warnf("CPU planning: no demand set from zedmanager yet; planning over " + + "the DomainConfigs received so far, which makes the layout depend on " + + "the order workloads activate in") + if ctx.subDomainConfig == nil { + return nil + } + var intents []cpuIntent + for _, item := range ctx.subDomainConfig.GetAll() { + if config, ok := item.(types.DomainConfig); ok { + intents = append(intents, intentOfConfig(&config)) + } + } + return intents +} + +// planPinnedPlacement computes the placement plan for every pinned workload the +// controller has configured, whether or not it is running yet. +// +// Planning the whole set, rather than allocating for each workload as it +// activates, is what makes the outcome independent of the order workloads start +// in -- including a workload with a start delay, whose CPUs must still be +// waiting for it when it eventually starts. +// +// The plan is recomputed rather than remembered: it is a pure function of the +// configured set and the topology, so recomputing yields the same answer, and +// there is no stored copy to go stale or to have to migrate. +func planPinnedPlacement(ctx *domainContext) map[uuid.UUID]cpuallocator.Result { + if ctx.placer == nil { + return nil + } + var requests []cpuallocator.Request + for _, intent := range plannedIntents(ctx) { + // The demand set carries the controller's intent only -- zedmanager + // cannot see the operator's /persist override -- so whether a workload + // is pinned, and how, is still resolved here. + if !cpuIntentPinned(intent) { + continue + } + placement, err := placementForIntent(intent) + if err == nil { + err = validateVCPUCount(placement, intent.vcpus) + } + if err != nil { + // A workload whose intent cannot be satisfied is not part of the + // plan, and its own activation is what reports that to the + // controller. It is still logged here, because leaving it out also + // changes where every other workload lands, and the plan file would + // otherwise be unexplainable. + log.Warnf("CPU planning: %s is left out of the plan: %v", + intent.displayName, err) + continue + } + // Thread-granular workloads are planned too (as ModeShared). Nothing + // applies their planned assignment -- they are still allocated on + // arrival -- but they do take CPUs exclusively, so the housekeeping set + // derived from this plan has to account for them. + requests = append(requests, cpuallocator.Request{ + UUID: intent.id, + NumVCPUs: intent.vcpus, + Mode: placement.Mode, + NUMA: placement.NUMA, + }) + } + if len(requests) == 0 { + return nil + } + return ctx.placer.Plan(requests) +} + +// claimPlannedPlacement returns the planned assignment for a workload if it can +// be taken as planned, i.e. every CPU the plan set aside for it is still free. +// +// A planned assignment is only usable while the plan and reality still agree. +// Workloads already running cannot be moved -- their vCPU threads are pinned and +// their guest was told a fixed topology at launch -- so if something else now +// holds a CPU this workload was planned onto, the plan cannot be applied as-is +// and the caller falls back to placing it among whatever is free. That fallback +// is the situation the optimality signal exists to report. +// +// vcpus is the count from the DomainConfig the workload is about to be created +// from. The plan is computed from zedmanager's demand set, which is a separate +// publication, so after a vCPU-count change the two can disagree for a moment. +// An assignment of the wrong size would be applied anyway and then contradict +// the -smp topology the guest is launched with, so QEMU would refuse to start +// and the retry would reuse the same stale assignment forever. +func claimPlannedPlacement(ctx *domainContext, id uuid.UUID, vcpus int, + plan map[uuid.UUID]cpuallocator.Result) *cpuallocator.Assignment { + result, planned := plan[id] + if !planned || result.Status != cpuallocator.Success || + result.Assignment == nil { + return nil + } + if len(result.Assignment.OrderedHostCPUs) != vcpus { + log.Noticef("CPU planning: the plan for %s is for %d vCPUs but its config "+ + "asks for %d; placing it among free CPUs instead", id, + len(result.Assignment.OrderedHostCPUs), vcpus) + return nil + } + held := map[uint32]bool{} + for _, cpu := range ctx.placer.DedicatedSet() { + held[uint32(cpu)] = true + } + for _, cpu := range result.Assignment.OrderedHostCPUs { + if held[uint32(cpu)] { + return nil + } + } + for _, cpu := range result.Assignment.ParkedCPUs { + if held[uint32(cpu)] { + return nil + } + } + return result.Assignment +} + +// plannedSlotBlockers names the workloads holding CPUs the plan set aside for +// another one, which is exactly the set that has to be restarted for a repack to +// free that slot. +// +// It is only for the report: "a repack would fix this" without naming who is in +// the way leaves the operator to work it out from the plan file, and nothing on +// the device restarts workloads by itself. Names are taken from the same set the +// plan was computed over, so a holder that has no DomainConfig here is still +// named; a holder that cannot be named at all is skipped rather than reported as +// a UUID nobody recognises. +func plannedSlotBlockers(ctx *domainContext, planned *cpuallocator.Assignment) []string { + if ctx.placer == nil || planned == nil { + return nil + } + names := map[uuid.UUID]string{} + for _, intent := range plannedIntents(ctx) { + names[intent.id] = intent.displayName + } + var blockers []string + seen := map[uuid.UUID]bool{} + for _, cpu := range assignmentCPUs(planned) { + id, held := ctx.placer.HolderOf(cputopology.LCPU(cpu)) + if !held || seen[id] { + continue + } + seen[id] = true + if name := names[id]; name != "" { + blockers = append(blockers, name) + } + } + sort.Strings(blockers) + return blockers +} + +// emulatorHousekeepingCPUs is the CPU set a pinned VM's emulator/IO threads may +// be pinned to under io_placement=housekeeping. +// +// It deliberately is not "whatever is free right now". The set is chosen when +// the VM activates and is never revisited for a pinned VM -- only non-pinned VMs +// get their cpuset redistributed -- so a workload that starts later and takes +// CPUs of its own would find this VM's emulator threads already running on them, +// which is exactly the interference dedicated CPUs exist to prevent. +// +// The CPUs reserved for EVE's own services are therefore preferred: no workload +// can ever be given one, so a set drawn from them cannot be invalidated by a +// later deployment. Only when the node reserves none does this fall back to the +// CPUs no *currently configured* pinned workload was planned onto -- which is +// stable against start order but not against a workload deployed later, and is +// still better than leaving the emulator threads on the hot vCPU cores. +func emulatorHousekeepingCPUs(ctx *domainContext, plan map[uuid.UUID]cpuallocator.Result) []uint32 { + if reserved := reservedForEVECPUs(ctx); len(reserved) > 0 { + return reserved + } + planned := map[uint32]bool{} + for _, result := range plan { + if result.Status != cpuallocator.Success || result.Assignment == nil { + continue + } + for _, cpu := range assignmentCPUs(result.Assignment) { + planned[cpu] = true + } + } + var out []uint32 + for _, cpu := range housekeepingCPUs(ctx) { + if !planned[cpu] { + out = append(out, cpu) + } + } + return out +} + +// reservedForEVECPUs is the low range of logical CPUs withheld from workloads, +// where EVE's own services run. The allocator applies the same lower bound, so +// these CPUs are the only ones guaranteed to stay free of every workload for as +// long as the node runs. +func reservedForEVECPUs(ctx *domainContext) []uint32 { + var out []uint32 + for cpu := uint32(0); cpu < ctx.cpusReserved; cpu++ { + out = append(out, cpu) + } + return out +} + +// publishCPUPlan mirrors the plan to /run for inspection. Failures are not +// fatal: the plan is a diagnostic, and refusing to place a workload because its +// description could not be written would be worse than not writing it. +func publishCPUPlan(ctx *domainContext, plan map[uuid.UUID]cpuallocator.Result) { + if plan == nil { + _ = os.Remove(cpuPlanFile) + return + } + // Described from the same set the plan was computed over, so a workload + // that is planned but has no DomainConfig yet is still named here. + names := map[uuid.UUID]string{} + modes := map[uuid.UUID]string{} + vcpus := map[uuid.UUID]int{} + for _, intent := range plannedIntents(ctx) { + names[intent.id] = intent.displayName + vcpus[intent.id] = intent.vcpus + if placement, err := placementForIntent(intent); err == nil { + modes[intent.id] = placement.Mode.String() + } + } + + out := cpuPlan{ + Comment: "Derived CPU placement plan, recomputed from the app configs and " + + "the CPU topology. Diagnostic only: nothing reads this file back.", + } + for id, result := range plan { + entry := plannedPlacement{ + UUID: id.String(), + DisplayName: names[id], + Mode: modes[id], + VCPUs: vcpus[id], + Status: result.Status.String(), + Message: result.Message, + } + if result.Assignment != nil { + for _, cpu := range result.Assignment.OrderedHostCPUs { + entry.HostCPUs = append(entry.HostCPUs, uint32(cpu)) + } + for _, cpu := range result.Assignment.ParkedCPUs { + entry.ParkedCPUs = append(entry.ParkedCPUs, uint32(cpu)) + } + } + out.Workloads = append(out.Workloads, entry) + } + // Stable output so a diff between two boots is meaningful. + sort.Slice(out.Workloads, func(i, j int) bool { + return out.Workloads[i].UUID < out.Workloads[j].UUID + }) + + data, err := json.MarshalIndent(out, "", " ") + if err != nil { + log.Errorf("publishCPUPlan: marshal failed: %v", err) + return + } + tmp := cpuPlanFile + ".tmp" + if err := os.WriteFile(tmp, data, 0644); err != nil { + log.Errorf("publishCPUPlan: write failed: %v", err) + return + } + if err := os.Rename(tmp, cpuPlanFile); err != nil { + log.Errorf("publishCPUPlan: rename failed: %v", err) + } +} + +// assignmentCPUs is every CPU an assignment occupies: the ones backing vCPUs +// plus the siblings it parks. Parked siblings count as occupied -- that is the +// point of asking for whole cores -- so they must be reserved too, or another +// workload would be free to take them. +func assignmentCPUs(a *cpuallocator.Assignment) []uint32 { + cpus := make([]uint32, 0, len(a.OrderedHostCPUs)+len(a.ParkedCPUs)) + for _, cpu := range a.OrderedHostCPUs { + cpus = append(cpus, uint32(cpu)) + } + for _, cpu := range a.ParkedCPUs { + cpus = append(cpus, uint32(cpu)) + } + return cpus +} diff --git a/pkg/pillar/cmd/domainmgr/cpuplan_test.go b/pkg/pillar/cmd/domainmgr/cpuplan_test.go new file mode 100644 index 00000000000..ab895a23645 --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/cpuplan_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "testing" + + "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/cputopology" + uuid "github.com/satori/go.uuid" +) + +func plannedOn(cpus ...cputopology.LCPU) cpuallocator.Result { + return cpuallocator.Result{ + Status: cpuallocator.Success, + Assignment: &cpuallocator.Assignment{OrderedHostCPUs: cpus}, + } +} + +// Regression: the emulator housekeeping set is computed once, when the VM +// activates, and is never recomputed for a pinned VM. Deriving it from what +// happens to be free at that moment therefore hands VM1's emulator/IO threads +// the very CPUs VM2 later takes as its dedicated cores. +func TestEmulatorHousekeepingCPUs_ExcludesPlannedPinnedCPUs(t *testing.T) { + placer := testPlacer(t) // 4 SMT2 cores: {0,4} {1,5} {2,6} {3,7} + ctx := &domainContext{placer: placer} + + running := uuid.NewV5(uuid.NamespaceOID, "running") + notStartedYet := uuid.NewV5(uuid.NamespaceOID, "later") + placer.Reserve(running, []uint32{0, 4}) + + plan := map[uuid.UUID]cpuallocator.Result{ + running: plannedOn(0, 4), + notStartedYet: plannedOn(1, 5), + } + + // The free set alone still offers the CPUs the second workload is planned + // onto -- that is the leak. + if free := housekeepingCPUs(ctx); !contains(free, 1) || !contains(free, 5) { + t.Fatalf("precondition: the free set should still contain 1 and 5, got %v", free) + } + + got := emulatorHousekeepingCPUs(ctx, plan) + for _, cpu := range []uint32{0, 1, 4, 5} { + if contains(got, cpu) { + t.Errorf("housekeeping set %v must not contain pinned CPU %d", got, cpu) + } + } + for _, cpu := range []uint32{2, 3, 6, 7} { + if !contains(got, cpu) { + t.Errorf("housekeeping set %v should still offer unclaimed CPU %d", got, cpu) + } + } +} + +// A workload the plan could not place holds nothing, so it must not shrink the +// housekeeping set. +func TestEmulatorHousekeepingCPUs_IgnoresUnplaceableWorkloads(t *testing.T) { + ctx := &domainContext{placer: testPlacer(t)} + plan := map[uuid.UUID]cpuallocator.Result{ + uuid.NewV5(uuid.NamespaceOID, "toobig"): { + Status: cpuallocator.Insufficient, Message: "need 8 free cores, have 4", + }, + } + if got := emulatorHousekeepingCPUs(ctx, plan); len(got) != 8 { + t.Errorf("want all 8 CPUs available for housekeeping, got %v", got) + } +} + +func contains(cpus []uint32, want uint32) bool { + for _, cpu := range cpus { + if cpu == want { + return true + } + } + return false +} diff --git a/pkg/pillar/cmd/domainmgr/cpupools.go b/pkg/pillar/cmd/domainmgr/cpupools.go new file mode 100644 index 00000000000..8aafc1edf98 --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/cpupools.go @@ -0,0 +1,87 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/cputopology" + "github.com/lf-edge/eve/pkg/pillar/hardware" + "github.com/lf-edge/eve/pkg/pillar/types" +) + +// readIsolatedCPUs returns the logical CPUs the running kernel isolates. +// +// Read from sysfs rather than from the kernel command line, so it reports what +// the kernel is actually doing. It cannot change without a reboot, so it is read +// once and kept. +func readIsolatedCPUs() []cputopology.LCPU { + isolated, _, _ := hardware.IsolatedCPUSets() + out := make([]cputopology.LCPU, 0, len(isolated)) + for _, cpu := range isolated { + out = append(out, cputopology.LCPU(cpu)) + } + return out +} + +// cpuPoolKind maps an allocator pool onto the published vocabulary. +func cpuPoolKind(pool cpuallocator.CPUPool) types.CPUPoolKind { + switch pool { + case cpuallocator.PoolHousekeeping: + return types.CPUPoolKindHousekeeping + case cpuallocator.PoolDedicated: + return types.CPUPoolKindDedicated + case cpuallocator.PoolIsolated: + return types.CPUPoolKindIsolated + } + // A pool with real CPUs in it and no kind reads on the wire as capacity + // nobody can account for, so a newly added pool must not slip out unlabelled. + log.Errorf("cpuPoolKind: allocator pool %d has no published kind", pool) + return types.CPUPoolKindUnspecified +} + +// cpuPoolStatus projects the allocator's view of the node's CPU pools onto the +// published type. +func cpuPoolStatus(placer *cpuallocator.Placer, isolated []cputopology.LCPU) types.CPUPoolStatus { + var out types.CPUPoolStatus + for _, pool := range placer.PoolUtilization(isolated) { + out.Pools = append(out.Pools, types.CPUPoolUtilization{ + Kind: cpuPoolKind(pool.Pool), + CPUs: lcpusToUint32(pool.CPUs), + FreeCPUs: lcpusToUint32(pool.FreeCPUs), + TotalThreads: pool.TotalThreads, + AllocatedThreads: pool.AllocatedThreads, + FreeThreads: pool.FreeThreads, + TotalCores: pool.TotalCores, + FreeWholeCores: pool.FreeWholeCores, + }) + } + return out +} + +func lcpusToUint32(cpus []cputopology.LCPU) []uint32 { + if len(cpus) == 0 { + return nil + } + out := make([]uint32, 0, len(cpus)) + for _, cpu := range cpus { + out = append(out, uint32(cpu)) + } + return out +} + +// publishCPUPoolStatus recomputes the node CPU pool report and publishes it. +// +// The report is derived state, so it is recomputed rather than incrementally +// maintained, and this is called from every path that touches the allocator. +// pubsub drops a republication that is byte-identical to the last one, so +// calling it on a path that changed nothing costs nothing. +func publishCPUPoolStatus(ctx *domainContext) { + if ctx.placer == nil || ctx.pubCPUPoolStatus == nil { + return + } + status := cpuPoolStatus(ctx.placer, ctx.isolatedCPUs) + if err := ctx.pubCPUPoolStatus.Publish(status.Key(), status); err != nil { + log.Errorf("publishCPUPoolStatus failed: %v", err) + } +} diff --git a/pkg/pillar/cmd/domainmgr/cpupools_test.go b/pkg/pillar/cmd/domainmgr/cpupools_test.go new file mode 100644 index 00000000000..b5987083e78 --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/cpupools_test.go @@ -0,0 +1,137 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "path/filepath" + "reflect" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/cputopology" + "github.com/lf-edge/eve/pkg/pillar/pubsub" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +func poolOfKind(t *testing.T, status types.CPUPoolStatus, + kind types.CPUPoolKind) types.CPUPoolUtilization { + t.Helper() + for _, pool := range status.Pools { + if pool.Kind == kind { + return pool + } + } + t.Fatalf("no %s pool in %+v", kind, status.Pools) + return types.CPUPoolUtilization{} +} + +func publishedPools(t *testing.T, pub pubsub.Publication) types.CPUPoolStatus { + t.Helper() + item, err := pub.Get("global") + if err != nil { + t.Fatalf("the pool report was not published: %v", err) + } + status, ok := item.(types.CPUPoolStatus) + if !ok { + t.Fatalf("unexpected published type %T", item) + } + return status +} + +// The pool report is what a controller uses to answer "will this workload fit?", +// so it has to be published, cover every pool, and say how many whole cores are +// left -- not just how many threads. +func TestPublishCPUPoolStatus(t *testing.T) { + ps := testPubSub(t) + pub := testPublication(t, ps, types.CPUPoolStatus{}) + ctx := &domainContext{ + placer: testPlacer(t), + pubCPUPoolStatus: pub, + isolatedCPUs: []cputopology.LCPU{2, 6}, + } + // testPlacer numbers the siblings of core c as CPU c and CPU c+4. + ctx.placer.Reserve(uuid.NewV5(uuid.NamespaceOID, "pinned"), []uint32{0, 4}) + + publishCPUPoolStatus(ctx) + status := publishedPools(t, pub) + + dedicated := poolOfKind(t, status, types.CPUPoolKindDedicated) + if !reflect.DeepEqual(dedicated.CPUs, []uint32{0, 4}) { + t.Errorf("dedicated CPUs: want [0 4], got %v", dedicated.CPUs) + } + if dedicated.TotalCores != 1 || dedicated.FreeWholeCores != 0 { + t.Errorf("one whole core is taken: want 1/0 total/free cores, got %d/%d", + dedicated.TotalCores, dedicated.FreeWholeCores) + } + + housekeeping := poolOfKind(t, status, types.CPUPoolKindHousekeeping) + if housekeeping.FreeThreads != 6 || housekeeping.FreeWholeCores != 3 { + t.Errorf("want 6 free threads on 3 free whole cores, got %d/%d", + housekeeping.FreeThreads, housekeeping.FreeWholeCores) + } + + isolated := poolOfKind(t, status, types.CPUPoolKindIsolated) + if !reflect.DeepEqual(isolated.CPUs, []uint32{2, 6}) { + t.Errorf("isolated CPUs: want the kernel's set [2 6], got %v", isolated.CPUs) + } + if isolated.FreeWholeCores != 1 { + t.Errorf("core 2 is isolated and untaken, want 1 free whole core, got %d", + isolated.FreeWholeCores) + } +} + +// Taking cores away from the shared pool has to move the report with it: a +// stale report would keep promising capacity that is already spoken for. +func TestPublishCPUPoolStatus_FollowsAllocation(t *testing.T) { + isolatePinningOverride(t) + cpuPlanFile = filepath.Join(t.TempDir(), "cpuplan.json") + config := pinnedConfigForTest("wholecore", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }) + + ps := testPubSub(t) + pub := testPublication(t, ps, types.CPUPoolStatus{}) + ctx := &domainContext{ + placer: testPlacer(t), + pubCPUPoolStatus: pub, + subDomainConfig: testDomainConfigSub(t, ps, config), + cpuTopologyPinningSupported: true, + } + var status types.DomainStatus + status.UUIDandVersion = config.UUIDandVersion + + freeCores := func() uint32 { + return poolOfKind(t, publishedPools(t, pub), + types.CPUPoolKindHousekeeping).FreeWholeCores + } + + if err := assignCPUs(ctx, &config, &status); err != nil { + t.Fatalf("assignCPUs: %v", err) + } + if got := freeCores(); got != 3 { + t.Errorf("one core was handed out, want 3 free whole cores, got %d", got) + } + + releaseCPUs(ctx, &status) + if got := freeCores(); got != 4 { + t.Errorf("the core was given back, want 4 free whole cores, got %d", got) + } +} + +// Releasing a workload's CPUs must also drop the verdict on the placement they +// were part of, or a stale "needs repack" would be reported for a workload that +// holds nothing. +func TestReleaseCPUs_ClearsPlacementQuality(t *testing.T) { + ctx := &domainContext{placer: testPlacer(t)} + status := types.DomainStatus{ + PlacementQuality: types.CPUPlacementQualityNeedsRepack, + } + status.VmConfig.CPUs = []uint32{1, 5} + + releaseCPUs(ctx, &status) + + if status.PlacementQuality != types.CPUPlacementQualityUnspecified { + t.Errorf("want the quality cleared, got %v", status.PlacementQuality) + } +} diff --git a/pkg/pillar/cmd/domainmgr/cpuredistribute_test.go b/pkg/pillar/cmd/domainmgr/cpuredistribute_test.go new file mode 100644 index 00000000000..5111a9d294a --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/cpuredistribute_test.go @@ -0,0 +1,160 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "reflect" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +// The cpuset a non-pinned workload is actually confined to is rewritten in the +// cgroup, but the record of it lives in DomainStatus -- which is what the rest +// of the device, and through zedagent the controller, reads. Without a publish +// the two drift apart: the workload is observed on two CPUs while EVE reports +// six. +func TestUpdateNonPinnedCPUs_PublishesTheEnforcedSet(t *testing.T) { + ps := testPubSub(t) + pub := testPublication(t, ps, types.DomainStatus{}) + ctx := &domainContext{placer: testPlacer(t), pubDomainStatus: pub} + // testPlacer numbers the siblings of core c as CPU c and CPU c+4, so this + // takes cores 2 and 3 away from the shared pool. + ctx.placer.Reserve(uuid.NewV5(uuid.NamespaceOID, "pinned"), + []uint32{2, 3, 6, 7}) + + config := sharedConfigForTest("besteffort") + status := types.DomainStatus{UUIDandVersion: config.UUIDandVersion} + status.DomainName = config.DisplayName + // The set from before the pinned workload arrived. + status.VmConfig.CPUs = []uint32{0, 1, 2, 3, 4, 5, 6, 7} + if err := pub.Publish(status.Key(), status); err != nil { + t.Fatalf("Publish: %v", err) + } + + if err := updateNonPinnedCPUs(ctx, &config, &status); err != nil { + t.Fatalf("updateNonPinnedCPUs: %v", err) + } + + want := []uint32{0, 1, 4, 5} + if got := publishedCPUs(t, pub, status.Key()); !reflect.DeepEqual(got, want) { + t.Errorf("published CPUs %v, want %v -- the record must describe the "+ + "cpuset that is enforced, not the one from activation time", got, want) + } +} + +// A wakeup that arrives when the workload cannot be updated -- here, before it +// has a DomainStatus at all -- must leave the change pending rather than +// consume it. Nothing sends a second wakeup for a change that already happened, +// so a swallowed one leaves the workload spread across cores another workload +// now owns. +func TestRedistributeNonPinnedCPUs_UnappliedChangeStaysPending(t *testing.T) { + isolatePinningOverride(t) + ps := testPubSub(t) + statusPub := testPublication(t, ps, types.DomainStatus{}) + config := sharedConfigForTest("besteffort") + ctx := &domainContext{ + placer: testPlacer(t), + pubDomainStatus: statusPub, + subDomainConfig: testDomainConfigSub(t, ps, config), + cpuPinningSupported: true, + } + ctx.placer.Reserve(uuid.NewV5(uuid.NamespaceOID, "pinned"), + []uint32{2, 3, 6, 7}) + + applied := cpuAllocationGen.Load() + triggerCPUNotification() + + // No DomainStatus yet: there is nothing to update, and the change must not + // be marked as applied. + if got := redistributeNonPinnedCPUs(ctx, config.Key(), applied); got != applied { + t.Fatalf("an un-applied change must stay pending, generation moved to %d", got) + } + + status := types.DomainStatus{UUIDandVersion: config.UUIDandVersion} + status.DomainName = config.DisplayName + status.VmConfig.CPUs = []uint32{0, 1, 2, 3, 4, 5, 6, 7} + if err := statusPub.Publish(status.Key(), status); err != nil { + t.Fatalf("Publish: %v", err) + } + + // The catch-up pass has to do the work with no new wakeup. + nowApplied := redistributeNonPinnedCPUs(ctx, config.Key(), applied) + if nowApplied == applied { + t.Fatal("the pending change was never applied") + } + want := []uint32{0, 1, 4, 5} + if got := publishedCPUs(t, statusPub, status.Key()); !reflect.DeepEqual(got, want) { + t.Errorf("cpuset %v, want %v", got, want) + } + + // And with nothing left pending it must not churn the cgroup on every tick. + if got := redistributeNonPinnedCPUs(ctx, config.Key(), nowApplied); got != nowApplied { + t.Errorf("nothing changed, generation should stay at %d, got %d", + nowApplied, got) + } +} + +// A pinned workload keeps the CPUs it was given, so a redistribution wakeup is +// nothing for it to do -- but it must still count as handled, or every timer +// tick would retry it forever. +func TestRedistributeNonPinnedCPUs_PinnedWorkloadUntouched(t *testing.T) { + isolatePinningOverride(t) + ps := testPubSub(t) + statusPub := testPublication(t, ps, types.DomainStatus{}) + config := pinnedConfigForTest("pinned", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }) + ctx := &domainContext{ + placer: testPlacer(t), + pubDomainStatus: statusPub, + subDomainConfig: testDomainConfigSub(t, ps, config), + cpuPinningSupported: true, + } + status := types.DomainStatus{UUIDandVersion: config.UUIDandVersion} + status.DomainName = config.DisplayName + status.VmConfig.CPUs = []uint32{1, 5} + if err := statusPub.Publish(status.Key(), status); err != nil { + t.Fatalf("Publish: %v", err) + } + + applied := cpuAllocationGen.Load() + triggerCPUNotification() + current := cpuAllocationGen.Load() + + if got := redistributeNonPinnedCPUs(ctx, config.Key(), applied); got != current { + t.Errorf("a pinned workload has nothing pending, want generation %d, got %d", + current, got) + } + if got := publishedCPUs(t, statusPub, status.Key()); !reflect.DeepEqual(got, []uint32{1, 5}) { + t.Errorf("a pinned workload must keep its own CPUs, got %v", got) + } +} + +// Coalescing the wakeups is fine; losing the fact that something changed is +// not. Two changes in a row leave only one wakeup queued, so the generation is +// what has to carry the second one. +func TestTriggerCPUNotification_CoalescedWakeupStillCounts(t *testing.T) { + handlersInit() + defer handlersInit() + cpuChannel := make(chan Notify, 1) + handlerMap["domain"] = channels{ + configChannel: make(chan Notify, 1), + cpuChannel: cpuChannel, + } + + before := cpuAllocationGen.Load() + triggerCPUNotification() + triggerCPUNotification() + + if len(cpuChannel) != 1 { + t.Errorf("want the second wakeup coalesced into the queued one, got %d queued", + len(cpuChannel)) + } + if got := cpuAllocationGen.Load(); got != before+2 { + t.Errorf("both changes must be counted even though one wakeup was "+ + "dropped: generation %d, want %d", got, before+2) + } +} diff --git a/pkg/pillar/cmd/domainmgr/domainmgr.go b/pkg/pillar/cmd/domainmgr/domainmgr.go index 5131b5b373a..73449d103ec 100644 --- a/pkg/pillar/cmd/domainmgr/domainmgr.go +++ b/pkg/pillar/cmd/domainmgr/domainmgr.go @@ -23,6 +23,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/containerd/cgroups" @@ -38,6 +39,7 @@ import ( "github.com/lf-edge/eve/pkg/pillar/cipher" "github.com/lf-edge/eve/pkg/pillar/containerd" "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/cputopology" "github.com/lf-edge/eve/pkg/pillar/flextimer" "github.com/lf-edge/eve/pkg/pillar/hypervisor" "github.com/lf-edge/eve/pkg/pillar/kubeapi" @@ -151,8 +153,30 @@ type domainContext struct { // cli options hypervisorPtr *string // CPUs management - cpuAllocator *cpuallocator.CPUAllocator + placer *cpuallocator.Placer cpuPinningSupported bool + // subCPUDemandSet is zedmanager's set of apps intended to run, with their + // CPU intent. Placement is planned over it rather than over the + // DomainConfigs received so far, which cover only the apps whose volumes + // already resolved. + subCPUDemandSet pubsub.Subscription + // pubCPUPoolStatus carries the node CPU pool report to zedagent, which + // projects it onto ZInfoDevice.cpu_pools. domainmgr owns the allocator, so + // it is the only agent that can compute it. + pubCPUPoolStatus pubsub.Publication + // isolatedCPUs is what the running kernel isolates (isolcpus), read once at + // startup: it can only change across a reboot. + isolatedCPUs []cputopology.LCPU + // cpuTopologyDegraded is set when the CPU topology could not be read from + // sysfs and had to be synthesized. Whole-core placement is refused while it + // is set, since the model cannot tell SMT siblings apart. + cpuTopologyDegraded bool + // cpusReserved is how many low-numbered logical CPUs are withheld from + // workloads for EVE's own services. + cpusReserved uint32 + // cpuTopologyPinningSupported is whether the hypervisor can actually apply + // a whole-core placement (per-vCPU pinning + guest SMT topology). + cpuTopologyPinningSupported bool // Is it EVE 'k' hvTypeKube bool nodeName string @@ -341,6 +365,15 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar } domainCtx.pubCapabilities = capabilitiesInfoPub + cpuPoolStatusPub, err := ps.NewPublication(pubsub.PublicationOptions{ + AgentName: agentName, + TopicType: types.CPUPoolStatus{}, + }) + if err != nil { + log.Fatal(err) + } + domainCtx.pubCPUPoolStatus = cpuPoolStatusPub + // Look for nodeagent status subNodeAgentStatus, err := ps.NewSubscription(pubsub.SubscriptionOptions{ AgentName: "nodeagent", @@ -470,6 +503,24 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar } domainCtx.subZFSPoolStatus = subZFSPoolStatus + // Subscribed and activated well before DomainConfig, so the demand set is + // already in hand when the first workload asks to be placed. A DomainConfig + // that overtakes it only costs an order-dependent layout, not a failure to + // start, but there is no reason to invite it. + subCPUDemandSet, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "zedmanager", + MyAgentName: agentName, + TopicImpl: types.CPUDemandSet{}, + Activate: true, + Ctx: &domainCtx, + WarningTime: warningTime, + ErrorTime: errorTime, + }) + if err != nil { + log.Fatal(err) + } + domainCtx.subCPUDemandSet = subCPUDemandSet + // Wait for ConfigItemValueMap for !domainCtx.GCComplete { log.Noticef("waiting for GCComplete") @@ -580,6 +631,9 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar case change := <-subZFSPoolStatus.MsgChan(): subZFSPoolStatus.ProcessChange(change) + case change := <-subCPUDemandSet.MsgChan(): + subCPUDemandSet.ProcessChange(change) + case <-domainCtx.publishTicker.C: publishProcessesHandler(&domainCtx) @@ -605,12 +659,12 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar log.Fatal(err) } domainCtx.cpuPinningSupported = caps.CPUPinning + domainCtx.cpuTopologyPinningSupported = caps.CPUTopologyPinning // Need to wait for things to get started - var resources types.HostMemory for i := 0; true; i++ { delay := 10 - resources, err = hyper.GetHostCPUMem() + _, err = hyper.GetHostCPUMem() if err == nil { break } @@ -627,9 +681,35 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar log.Warnf("Failed to get reserved CPU number, use 1 by default: %s", err) } - if domainCtx.cpuAllocator, err = cpuallocator.Init(resources.Ncpus, uint32(cpusReserved)); err != nil { - log.Fatal(err) + topo, terr := cputopology.DiscoverTopology() + if terr != nil { + // The fallback model claims every logical CPU is a single-thread core, + // so it cannot say which CPUs are SMT siblings. Placement is refused on + // it rather than performed against a fabricated topology: a one-per-core + // workload would park nothing and get siblings of cores another workload + // is using, while being reported as optimally placed. + log.Errorf("CPU topology discovery failed (%v); the topology model is "+ + "synthetic, so whole-core CPU placement is refused on this node", terr) } + domainCtx.cpuTopologyDegraded = topo.Degraded + domainCtx.placer, err = cpuallocator.NewPlacer(topo, uint32(cpusReserved)) + if err != nil { + log.Fatalf("Cannot allocate CPUs (check the eve_max_vcpus kernel argument): %s", err) + } + domainCtx.cpusReserved = uint32(cpusReserved) + log.Noticef("CPU topology: %d physical cores", len(topo.Cores)) + domainCtx.isolatedCPUs = readIsolatedCPUs() + if len(domainCtx.isolatedCPUs) > 0 { + log.Noticef("Kernel isolates CPUs %v", domainCtx.isolatedCPUs) + } + // Reseed the allocator with cores already held by running VMs. DomainStatus + // is ephemeral (/run), so this only matters across a domainmgr process + // restart (not a device reboot); it prevents handing a running VM's + // dedicated cores to another VM before the existing status is reconciled. + seedPlacerFromStatus(&domainCtx) + // Report the pools right away: on a node with no pinned workload at all this + // is the only publication, and the controller still needs the node CPU map. + publishCPUPoolStatus(&domainCtx) // Wait until we have been onboarded aka know our own UUID however we do not use the UUID if _, err := wait.WaitForOnboarded(ps, log, agentName, warningTime, errorTime); err != nil { @@ -797,6 +877,9 @@ func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, ar case change := <-subDomainConfig.MsgChan(): subDomainConfig.ProcessChange(change) + case change := <-subCPUDemandSet.MsgChan(): + subCPUDemandSet.ProcessChange(change) + case change := <-subDeviceNetworkStatus.MsgChan(): subDeviceNetworkStatus.ProcessChange(change) @@ -933,16 +1016,43 @@ type handlers map[string]channels var handlerMap handlers +// handlerMapLock guards handlerMap. The map is created and torn down from the +// pubsub goroutine (handleDomainCreate/Delete) but read from every per-domain +// goroutine, because triggerCPUNotification fans out from whichever domain just +// took or gave back CPUs. Without the lock that is both a map race and a send +// on a channel handleDomainDelete may have just closed. +var handlerMapLock sync.Mutex + +// cpuAllocationGen counts changes to the set of CPUs dedicated to pinned +// workloads. Every non-pinned workload's cpuset is derived from that set, so +// each handler remembers the generation it last applied and recomputes whenever +// the counter has moved on. +// +// The wakeup channel is only a hint that it is worth looking; this counter is +// what makes the signal lossless. A wakeup dropped because one was already +// queued, or consumed at a moment the workload could not be updated (no status +// yet, crash-frozen, cgroup write failed), leaves the generation mismatched, so +// the work is still pending and gets done at the next wakeup or timer tick +// instead of being silently swallowed. +var cpuAllocationGen atomic.Uint64 + func handlersInit() { handlerMap = make(handlers) } +// triggerCPUNotification records that the dedicated CPU set changed and nudges +// every domain handler to reconsider its cpuset. Call it after the change is +// visible in the placer, never before. func triggerCPUNotification() { + cpuAllocationGen.Add(1) + handlerMapLock.Lock() + defer handlerMapLock.Unlock() for _, handler := range handlerMap { select { case handler.cpuChannel <- Notify{}: default: - log.Warnf("Already sent a CPU Notify...") + // A wakeup is already queued and the receiver re-reads the + // generation when it gets to it, so this one is redundant. } } } @@ -955,7 +1065,9 @@ func handleDomainModify(ctxArg interface{}, key string, configArg interface{}, log.Functionf("handleDomainModify(%s)", key) config := configArg.(types.DomainConfig) + handlerMapLock.Lock() h, ok := handlerMap[config.Key()] + handlerMapLock.Unlock() if !ok { log.Fatalf("handleDomainModify called on config that does not exist") } @@ -973,14 +1085,17 @@ func handleDomainCreate(ctxArg interface{}, key string, configArg interface{}) { log.Functionf("handleDomainCreate(%s)", key) ctx := ctxArg.(*domainContext) config := configArg.(types.DomainConfig) + handlerMapLock.Lock() h, ok := handlerMap[config.Key()] if ok { + handlerMapLock.Unlock() log.Fatalf("handleDomainCreate called on config that already exists") } hConfig := make(chan Notify, 1) hCPU := make(chan Notify, 1) h1 := channels{configChannel: hConfig, cpuChannel: hCPU} handlerMap[config.Key()] = h1 + handlerMapLock.Unlock() log.Functionf("Creating %s at %s", "runHandler", agentlog.GetMyStack()) go runHandler(ctx, key, hConfig, hCPU) h = h1 @@ -1005,13 +1120,16 @@ func handleDomainDelete(ctxArg interface{}, key string, unpublishCipherBlockStatus(ctx, config.Key()) } // Do we have a channel/goroutine? + handlerMapLock.Lock() h, ok := handlerMap[key] if ok { log.Functionf("Closing channels") close(h.cpuChannel) close(h.configChannel) delete(handlerMap, key) - } else { + } + handlerMapLock.Unlock() + if !ok { log.Tracef("handleDomainDelete: unknown %s", key) return } @@ -1035,6 +1153,11 @@ func runHandler(ctx *domainContext, key string, configChannel <-chan Notify, cpu // the capture goroutine. Buffered so that goroutine never blocks. captureDone := make(chan captureResult, 1) + // The CPU allocation generation this domain's cpuset was last derived from. + // Starting from the current value means a handler does not redistribute for + // changes that predate the domain, whose placement already accounted for them. + appliedCPUGen := cpuAllocationGen.Load() + closed := false for !closed { // Watch for a mode-A crash while the domain is activated and not already @@ -1077,29 +1200,7 @@ func runHandler(ctx *domainContext, key string, configChannel <-chan Notify, cpu } case _, ok := <-cpuChannel: if ok { - if !ctx.cpuPinningSupported { - continue - } - sub := ctx.subDomainConfig - c, err := sub.Get(key) - if err != nil { - log.Errorf("runHandler no config for %s", key) - continue - } - config := c.(types.DomainConfig) - status := lookupDomainStatus(ctx, key) - if status == nil { - log.Errorf("No Status for %s", config.DisplayName) - continue - } - if crashFrozen(status) { - continue - } - if !config.VmConfig.CPUsPinned { - if err = updateNonPinnedCPUs(ctx, &config, status); err != nil { - log.Warnf("failed to redistribute CPUs in %s", config.DisplayName) - } - } + appliedCPUGen = redistributeNonPinnedCPUs(ctx, key, appliedCPUGen) } case ev := <-crashCh: // Mode-A crash (guest internal-error): capture the guest core @@ -1126,11 +1227,60 @@ func runHandler(ctx *domainContext, key string, configChannel <-chan Notify, cpu // goroutine will drive finishCrashCapture; do not reconcile. } } + // Catch up on any CPU redistribution that could not be done when it + // was signalled -- the domain had no status yet, was crash-frozen, + // or the cgroup write failed. + appliedCPUGen = redistributeNonPinnedCPUs(ctx, key, appliedCPUGen) } } log.Functionf("runHandler(%s) DONE", key) } +// redistributeNonPinnedCPUs widens or narrows a non-pinned domain's cpuset to +// the CPUs no pinned workload holds, when the dedicated set has changed since +// this domain's cpuset was last derived from it. +// +// It returns the generation now applied. On any path that did not apply the +// current one it returns the caller's unchanged value, leaving the change +// pending so the next wakeup or timer tick retries it. +func redistributeNonPinnedCPUs(ctx *domainContext, key string, applied uint64) uint64 { + if !ctx.cpuPinningSupported { + return applied + } + // Read the generation before doing the work, so a change landing while this + // runs is not mistaken for one already applied. + current := cpuAllocationGen.Load() + if current == applied { + return applied + } + c, err := ctx.subDomainConfig.Get(key) + if err != nil { + log.Errorf("runHandler no config for %s", key) + return applied + } + config := c.(types.DomainConfig) + status := lookupDomainStatus(ctx, key) + if status == nil { + log.Errorf("No Status for %s", config.DisplayName) + return applied + } + if crashFrozen(status) { + return applied + } + // A pinned workload keeps its own CPUs. Handing it the shared set here would + // also fill status.VmConfig.CPUs, which assignCPUs reads as "already + // allocated" and would leave the workload permanently unpinned. Nothing is + // pending for it, so the generation counts as applied. + if effectiveCPUsPinned(&config) { + return current + } + if err := updateNonPinnedCPUs(ctx, &config, status); err != nil { + log.Warnf("failed to redistribute CPUs in %s: %v", config.DisplayName, err) + return applied + } + return current +} + // Check if it is still running func verifyStatus(ctx *domainContext, status *types.DomainStatus) { // Never reconcile a domain whose crash is being handled: a dump may be in @@ -1526,11 +1676,18 @@ func setCgroupCpuset(config *types.DomainConfig, status *types.DomainStatus) err } func updateNonPinnedCPUs(ctx *domainContext, config *types.DomainConfig, status *types.DomainStatus) error { - status.VmConfig.CPUs = ctx.cpuAllocator.GetAllFree() - err := setCgroupCpuset(config, status) - if err != nil { + previous := status.VmConfig.CPUs + status.VmConfig.CPUs = housekeepingCPUs(ctx) + if err := setCgroupCpuset(config, status); err != nil { + // The cpuset was not changed, so the record of it must not change either. + status.VmConfig.CPUs = previous return errors.New("failed to redistribute CPUs between VMs, can affect the inter-VM isolation") } + // DomainStatus is where everything else -- and, via zedagent, the controller + // -- reads which CPUs this workload runs on. Without publishing here it goes + // on describing the set from activation time while the cgroup enforces a + // narrower one. + publishDomainStatus(ctx, status) return nil } @@ -1538,38 +1695,300 @@ func updateNonPinnedCPUs(ctx *domainContext, config *types.DomainConfig, status // By the assignment, we mean that the CPUs are assigned in the CPUAllocator context to the given VM // and the cpumask is updated in the *status* func assignCPUs(ctx *domainContext, config *types.DomainConfig, status *types.DomainStatus) error { - if config.VmConfig.CPUsPinned { // Pin the CPU - // CPUs may already be allocated for this UUID from a prior - // doActivate that returned early (e.g. kubevirt cluster-trust path - // after a version bump). The cpuAllocator records one allocation - // per UUID and would error with "multiple allocations for UUID" - // on a second Allocate call, leaving the app permanently broken. - // Reuse the existing allocation when present. - if len(status.VmConfig.CPUs) > 0 { - return nil - } - cpusToAssign, err := ctx.cpuAllocator.Allocate(config.UUIDandVersion.UUID, config.VCpus) - if err != nil { - return err + err := allocateCPUs(ctx, config, status) + // The node CPU pool report is derived from the allocator, so refresh it + // whenever the allocator may have been touched -- including on the error + // paths, which can return after a reservation was already taken. + publishCPUPoolStatus(ctx) + return err +} + +// allocateCPUs is assignCPUs without the pool-report refresh, so every return +// path below can just return. +func allocateCPUs(ctx *domainContext, config *types.DomainConfig, status *types.DomainStatus) error { + if !effectiveCPUsPinned(config) { + // No pinning: the workload shares whatever no pinned workload holds. + status.VmConfig.CPUs = housekeepingCPUs(ctx) + return nil + } + // Record the effective decision: a /persist override can turn pinning on for + // a config the controller did not pin, and the release path, the post-restart + // reseed and the cpuset redistribution all read this flag from the status + // rather than re-deriving it. + status.VmConfig.CPUsPinned = true + // CPUs may already be allocated for this UUID from a prior doActivate that + // returned early. Reuse the existing allocation. + if len(status.VmConfig.CPUs) > 0 { + return nil + } + // Topology-aware placement, driven by the controller's policy when it sent + // one and by the /persist override otherwise. + placement, err := placementFor(config) + if err != nil { + return err + } + if err := validateVCPUCount(placement, config.VCpus); err != nil { + return err + } + if ctx.placer == nil { + return placementErrorf(types.ErrorCodeCPUTopologyUnsupported, + "CPU pinning for %s: no CPU allocator on this node", config.DisplayName) + } + // Refuse whole-core placement the hypervisor cannot carry out. Reserving the + // cores and filling in OrderedCPUs/VMTopology succeeds everywhere, but only + // kvm turns them into per-vCPU pinning and a guest SMT topology; anywhere + // else the workload would be reported as optimally placed while nothing was + // pinned at all. + if placement.TopologyAware && ctx.cpuTopologyDegraded { + return placementErrorf(types.ErrorCodeCPUTopologyUnsupported, + "whole-core CPU placement for %s: this node's CPU topology could not "+ + "be read, so SMT siblings cannot be identified", config.DisplayName) + } + if placement.TopologyAware && !ctx.cpuTopologyPinningSupported { + return placementErrorf(types.ErrorCodeCPUTopologyUnsupported, + "whole-core CPU placement for %s: the %s hypervisor cannot pin "+ + "individual vCPUs or expose a guest SMT topology", + config.DisplayName, hyper.Name()) + } + // Both paths place from the same plan, so a workload gets the same CPUs + // regardless of the order workloads start in. + plan := planPinnedPlacement(ctx) + publishCPUPlan(ctx, plan) + if placement.TopologyAware { + return allocateTopologyCPUs(ctx, config, status, placement, plan) + } + return allocateLegacyCPUs(ctx, config, status, plan) +} + +// allocateTopologyCPUs gives a workload whole physical cores, with the guest +// topology and per-vCPU order the hypervisor needs to pin them 1:1. +// +// It takes the CPUs the plan set aside for this workload whenever they are still +// free: the plan covers every configured pinned workload, not just the ones +// running, so a workload that starts late still finds its CPUs waiting rather +// than losing them to whoever started first. +func allocateTopologyCPUs(ctx *domainContext, config *types.DomainConfig, + status *types.DomainStatus, placement resolvedPlacement, + plan map[uuid.UUID]cpuallocator.Result) error { + log.Noticef("CPU pinning: %s requesting %d vCPUs, mode=%s numa=%s", + config.DisplayName, config.VCpus, placement.Mode, placement.NUMA) + + id := config.UUIDandVersion.UUID + a := claimPlannedPlacement(ctx, id, config.VCpus, plan) + if a != nil { + if err := ctx.placer.Reserve(id, assignmentCPUs(a)); err != nil { + return placementErrorf(types.ErrorCodeCPUPolicyInvalid, + "topology pinning for %s: reserving the planned CPUs failed: %v", + config.DisplayName, err) + } + status.PlacementQuality = types.CPUPlacementQualityOptimal + } else { + // The plan cannot be applied as-is, because something already running + // holds a CPU it wanted. Place this workload among whatever is free + // instead of moving a running one, which cannot be done safely. + res := ctx.placer.Allocate(cpuallocator.Request{ + UUID: id, + NumVCPUs: config.VCpus, + Mode: placement.Mode, + NUMA: placement.NUMA, + }) + if res.Status != cpuallocator.Success { + // The workload never starts, so it has no placement to rate. A + // quality here would have zedagent publish the "running, but could + // be packed better" advisory next to the fatal error, telling the + // controller the workload is up when it is not; the error code alone + // carries the repack verdict. + status.PlacementQuality = types.CPUPlacementQualityUnspecified + var blockers []string + if planned, ok := plan[id]; ok { + blockers = plannedSlotBlockers(ctx, planned.Assignment) + } + return liveAllocationError(config.DisplayName, id, res, plan, blockers) + } + a = res.Assignment + // Only a placement that is genuinely worse than the planned one warrants + // a repack. Landing on different CPUs of equal quality is not a problem, + // and reporting it as one would demand pointless restarts every time a + // workload's first-choice CPUs happened to be taken. + status.PlacementQuality = types.CPUPlacementQualityOptimal + if planned, ok := plan[id]; ok && planned.Status == cpuallocator.Success { + actual := ctx.placer.Score(a) + ideal := ctx.placer.Score(planned.Assignment) + if actual.WorseThan(ideal) { + status.PlacementQuality = types.CPUPlacementQualityNeedsRepack + log.Warnf("CPU pinning: %s placed on %v (spanning %d NUMA "+ + "node(s), %d L3 domain(s)) but the planned layout %v "+ + "would span %d/%d; a repack of the running workloads "+ + "would be needed to reach it", config.DisplayName, + a.OrderedHostCPUs, actual.NUMANodes, actual.L3Domains, + planned.Assignment.OrderedHostCPUs, ideal.NUMANodes, + ideal.L3Domains) + } } - for _, cpu := range cpusToAssign { - status.VmConfig.CPUs = append(status.VmConfig.CPUs, cpu) + if status.PlacementQuality == types.CPUPlacementQualityOptimal { + log.Noticef("CPU pinning: %s placed among free CPUs rather than "+ + "its planned slot, at equal quality", config.DisplayName) } - } else { // VM has no pinned CPUs, assign all the CPUs from the shared set - status.VmConfig.CPUs = ctx.cpuAllocator.GetAllFree() + } + log.Noticef("CPU pinning: %s allocated host CPUs %v (parked %v), guest topology sockets=%d cores=%d threads=%d", + config.DisplayName, a.OrderedHostCPUs, a.ParkedCPUs, a.Guest.Sockets, a.Guest.Cores, a.Guest.Threads) + for _, c := range a.OrderedHostCPUs { + status.VmConfig.CPUs = append(status.VmConfig.CPUs, uint32(c)) + } + for _, c := range a.ParkedCPUs { + status.VmConfig.CPUs = append(status.VmConfig.CPUs, uint32(c)) + } + for _, c := range a.OrderedHostCPUs { + status.OrderedCPUs = append(status.OrderedCPUs, uint32(c)) + } + status.VMTopology = types.CPUTopology{ + Sockets: a.Guest.Sockets, + Cores: a.Guest.Cores, + Threads: a.Guest.Threads, + } + if placement.IOHousekeeping { + applyIOHousekeeping(ctx, config, status, plan) } return nil } +// applyIOHousekeeping pins the QEMU main loop and iothread off the workload's +// hot vCPU cores, onto CPUs no pinned workload holds, and widens the cgroup +// cpuset so that pool is reachable. Under the default "dedicated" placement they +// stay on the workload's own cores and EmulatorCPUs is left nil, which is what +// tells pinDomainThreads to leave them alone. +func applyIOHousekeeping(ctx *domainContext, config *types.DomainConfig, + status *types.DomainStatus, plan map[uuid.UUID]cpuallocator.Result) { + hk := emulatorHousekeepingCPUs(ctx, plan) + if len(hk) == 0 { + // Nothing is safe from every configured pinned workload. Falling back to + // "dedicated" costs this workload some vCPU/IO separation; picking a CPU + // anyway would cost another workload the exclusive cores it was + // promised. The workload still runs, so this is reported as a placement + // that a repack could improve rather than as a failure -- otherwise the + // only trace of an unapplied request would be this log line. + log.Warnf("CPU pinning: %s asked for housekeeping IO placement but no "+ + "CPU is free of every pinned workload; leaving its emulator/IO "+ + "threads on its own cores", config.DisplayName) + status.PlacementQuality = types.CPUPlacementQualityNeedsRepack + return + } + status.EmulatorCPUs = hk + status.VmConfig.CPUs = append(status.VmConfig.CPUs, hk...) +} + +// allocateLegacyCPUs gives a workload exclusive CPUs at SMT-thread granularity, +// which is what a pin without a whole-core policy asks for. +// +// It claims the planned assignment for the same reason the topology path does. +// The plan ranks these workloads last but does account for them, and taking the +// lowest-numbered free CPUs instead -- which is what the allocator does when +// asked to place one in isolation -- takes threads off the very cores the plan +// set aside whole for someone else. Which of the two workloads then failed to +// start depended on which activated first. +func allocateLegacyCPUs(ctx *domainContext, config *types.DomainConfig, + status *types.DomainStatus, plan map[uuid.UUID]cpuallocator.Result) error { + id := config.UUIDandVersion.UUID + if a := claimPlannedPlacement(ctx, id, config.VCpus, plan); a != nil { + cpus := assignmentCPUs(a) + if err := ctx.placer.Reserve(id, cpus); err != nil { + return placementErrorf(types.ErrorCodeCPUPolicyInvalid, + "CPU pinning for %s: reserving the planned CPUs failed: %v", + config.DisplayName, err) + } + log.Noticef("CPU pinning: %s allocated its planned host CPUs %v (legacy, no policy)", + config.DisplayName, cpus) + status.VmConfig.CPUs = append(status.VmConfig.CPUs, cpus...) + return nil + } + cpusToAssign, err := ctx.placer.AllocateShared(id, config.VCpus) + if err != nil { + return err + } + log.Noticef("CPU pinning: %s allocated shared host CPUs %v (legacy, no policy)", + config.DisplayName, cpusToAssign) + for _, cpu := range cpusToAssign { + status.VmConfig.CPUs = append(status.VmConfig.CPUs, uint32(cpu)) + } + return nil +} + +// housekeepingCPUs returns all logical CPUs not dedicated to any VM +// (topology or shared), used for non-pinned VM cpusets and emulator pinning. +func housekeepingCPUs(ctx *domainContext) []uint32 { + var out []uint32 + for _, c := range ctx.placer.FreeCPUs() { + out = append(out, uint32(c)) + } + return out +} + // releaseCPUs releases the CPUs that were previously assigned to the VM. -// The cpumask in the *status* is updated accordingly, and the CPUs are released in the CPUAllocator context. -func releaseCPUs(ctx *domainContext, config *types.DomainConfig, status *types.DomainStatus) { - if ctx.cpuPinningSupported && config.VmConfig.CPUsPinned && status.VmConfig.CPUs != nil { - if err := ctx.cpuAllocator.Free(config.UUIDandVersion.UUID); err != nil { - log.Errorf("Failed to free CPUs for %s: %s", config.DisplayName, err) - } +// The cpumask in the *status* is updated accordingly, and the CPUs are released in the Placer. +func releaseCPUs(ctx *domainContext, status *types.DomainStatus) { + if ctx.placer != nil { + ctx.placer.Free(status.UUIDandVersion.UUID) } status.VmConfig.CPUs = nil + status.OrderedCPUs = nil + status.EmulatorCPUs = nil + status.VMTopology = types.CPUTopology{} + // CPUsPinned is part of the pin state, so it is cleared here too. It is set + // from the effective decision at allocation time, and a workload that is no + // longer pinned (the controller changed the policy, or the operator removed + // the override) would otherwise keep a stale true while VmConfig.CPUs holds + // the whole housekeeping set -- which the post-restart reseed reads as + // "these CPUs are exclusively mine" and reserves almost the entire node. + status.VmConfig.CPUsPinned = false + // A workload that holds no CPUs has no placement to judge. Leaving the old + // verdict behind would have it reported as "needs repack" long after the + // placement it described was given up. + status.PlacementQuality = types.CPUPlacementQualityUnspecified + publishCPUPoolStatus(ctx) +} + +// seedPlacerFromStatus reseeds the allocator with cores already held by running, +// pinned VMs, taken from the DomainStatus objects that survived in /run. A +// Placer created +// fresh on a domainmgr restart otherwise knows nothing of running VMs (the +// re-activate path reuses their existing status without re-running placement), +// and could hand their dedicated cores to another VM. Only the exclusive set is +// reserved: assigned CPUs minus the shared housekeeping/emulator pool. +func seedPlacerFromStatus(ctx *domainContext) { + if ctx.placer == nil { + return + } + for _, st := range ctx.pubDomainStatus.GetAll() { + status := st.(types.DomainStatus) + if !status.VmConfig.CPUsPinned || len(status.VmConfig.CPUs) == 0 { + continue + } + emu := make(map[uint32]bool, len(status.EmulatorCPUs)) + for _, c := range status.EmulatorCPUs { + emu[c] = true + } + var owned []uint32 + for _, c := range status.VmConfig.CPUs { + if !emu[c] { + owned = append(owned, c) + } + } + if len(owned) == 0 { + continue + } + // A rejection here means two statuses claim the same CPU, which no + // correct sequence of allocations can produce. Skipping the second + // claimant keeps the first workload's cores exclusive; double-booking + // them would leave both believing they own the core, and make the + // reported holder arbitrary. + if err := ctx.placer.Reserve(status.UUIDandVersion.UUID, owned); err != nil { + log.Errorf("CPU pinning: cannot reseed %s with dedicated CPUs %v: %v", + status.DisplayName, owned, err) + continue + } + log.Noticef("CPU pinning: reseeded %s with dedicated CPUs %v after domainmgr restart", + status.DisplayName, owned) + } } func handleCreate(ctx *domainContext, key string, config *types.DomainConfig) { @@ -1842,10 +2261,32 @@ func doActivate(ctx *domainContext, config types.DomainConfig, config.UUIDandVersion, config.DisplayName) if ctx.cpuPinningSupported { + ensureDomainInPinningConfig(config) if err := assignCPUs(ctx, &config, status); err != nil { log.Warnf("failed to assign CPUs for %s err %v", config.DisplayName, err) - errDescription := types.ErrorDescription{Error: err.Error()} - status.SetErrorDescription(errDescription) + // Deliberately neither BootFailed nor ConfigFailed: a workload + // refused for want of CPUs stays down until an explicit action + // (deactivate, which clears this error, then activate). That is how + // EVE already treats a workload whose assigned resource is not + // available -- an app that cannot get its PCI device is not brought + // up later just because the device reappeared -- and CPUs are an + // assigned resource like any other, so they behave the same way. + // + // This is a decision, not an oversight: do not wire this into + // maybeRetryBoot/maybeRetryConfig or into the CPU-change wakeup. + // Silently starting a workload minutes later, on CPUs the operator + // never saw it take, is the surprise this avoids. What the report + // owes instead is a retry_condition saying what to do, which is why + // placementErrorDescription always fills one in. + // + // AdaptersFailed is cleared for the same reason. It is the one + // failure flag whose retry path re-enters doActivate without + // clearing it first, so an adapter failure followed by a placement + // failure would have the timer re-attempt placement every tick -- + // and start the workload whenever some unrelated workload happened + // to release cores. + status.AdaptersFailed = false + status.SetErrorDescription(placementErrorDescription(err)) publishDomainStatus(ctx, status) return } @@ -1858,7 +2299,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, status.PendingAdd = false status.SetErrorDescription(*errDescription) status.AdaptersFailed = true - releaseCPUs(ctx, &config, status) + releaseCPUs(ctx, status) publishDomainStatus(ctx, status) releaseAdapters(ctx, config.IoAdapterList, config.UUIDandVersion.UUID, nil) @@ -1882,7 +2323,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, status.PendingAdd = false status.SetErrorNow(err.Error()) status.AdaptersFailed = true - releaseCPUs(ctx, &config, status) + releaseCPUs(ctx, status) publishDomainStatus(ctx, status) releaseAdapters(ctx, config.IoAdapterList, config.UUIDandVersion.UUID, nil) @@ -1906,7 +2347,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, snapshotID, config.UUIDandVersion.UUID, err) log.Error(err.Error()) status.SetErrorNow(err.Error()) - releaseCPUs(ctx, &config, status) + releaseCPUs(ctx, status) return } @@ -1935,7 +2376,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, err := fmt.Errorf("doActivate: Failed to write cloud-init metadata file. Error %s", err) log.Error(err.Error()) status.SetErrorNow(err.Error()) - releaseCPUs(ctx, &config, status) + releaseCPUs(ctx, status) return } @@ -1946,7 +2387,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, err := fmt.Errorf("doActivate: Failed to apply cloud-init config. Error %s", err) log.Error(err.Error()) status.SetErrorNow(err.Error()) - releaseCPUs(ctx, &config, status) + releaseCPUs(ctx, status) return } } @@ -1962,7 +2403,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, if err != nil { log.Errorf("Failed to check disk format: %v", err.Error()) status.SetErrorNow(err.Error()) - releaseCPUs(ctx, &config, status) + releaseCPUs(ctx, status) return } } @@ -2017,7 +2458,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, log.Errorf("Failed to create DomainStatus from %+v: %s", config, err) status.SetErrorNow(err.Error()) - releaseCPUs(ctx, &config, status) + releaseCPUs(ctx, status) return } @@ -2037,7 +2478,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, log.Errorf("DomainCreate for %s: %s", status.DomainName, err) status.BootFailed = true status.SetErrorNow(err.Error()) - releaseCPUs(ctx, &config, status) + releaseCPUs(ctx, status) publishDomainStatus(ctx, status) return } @@ -2053,7 +2494,7 @@ func doActivate(ctx *domainContext, config types.DomainConfig, status.SetErrorDescription(errDescription) publishDomainStatus(ctx, status) } - if config.CPUsPinned { + if status.VmConfig.CPUsPinned { triggerCPUNotification() } } @@ -2332,13 +2773,15 @@ func doCleanup(ctx *domainContext, status *types.DomainStatus) { } if ctx.cpuPinningSupported { - if status.VmConfig.CPUsPinned { - if err := ctx.cpuAllocator.Free(status.UUIDandVersion.UUID); err != nil { - log.Warnf("Failed to free for %s: %s", status.DisplayName, err) - } + wasPinned := status.VmConfig.CPUsPinned + // Release every CPU field (dedicated set, ordered vCPU map, emulator + // set, guest topology) through the single release path so a later + // re-activation starts from a clean slate and cannot accumulate stale + // pin state. + releaseCPUs(ctx, status) + if wasPinned { triggerCPUNotification() } - status.VmConfig.CPUs = nil } releaseAdapters(ctx, status.IoAdapterList, status.UUIDandVersion.UUID, status) diff --git a/pkg/pillar/cmd/domainmgr/pinningconfig.go b/pkg/pillar/cmd/domainmgr/pinningconfig.go new file mode 100644 index 00000000000..332147ce3ff --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/pinningconfig.go @@ -0,0 +1,235 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +// Paths are vars (not consts) so tests can point them at a temp dir. +var ( + pinConfigDir = "/persist/pinning" + pinConfigFile = "/persist/pinning/config.json" +) + +// PolicyOptions holds the K8s CPUManager policy-option modifiers on the static +// policy that EVE supports. +type PolicyOptions struct { + // FullPCPUsOnly allocates in whole-physical-core units: both SMT siblings + // of a core are owned by the VM and never shared with another workload. + FullPCPUsOnly bool `json:"full-pcpus-only"` +} + +// PinningEntry is one VM's operator-editable pinning policy. Field names follow +// the Kubernetes CPUManager / Topology Manager vocabulary so the config carries +// cleanly into the future EVE-K operator port. +type PinningEntry struct { + DisplayName string `json:"display_name"` + // UUID duplicates the map key on purpose: it survives display_name reuse and + // lets operators cross-reference and spot stale entries after redeployment. + UUID string `json:"uuid"` + // VCpus is recorded for the operator's benefit only: the effective count + // always comes from the app config, so editing it here changes nothing. + VCpus int `json:"vcpus"` + // CPUPolicy mirrors K8s CPUManager: "none" (shared CFS pool) or "static" + // (exclusive allocation). + CPUPolicy string `json:"cpu_policy"` + // PolicyOptions refine the static policy (K8s CPUManager policy options). + PolicyOptions *PolicyOptions `json:"policy_options,omitempty"` + // ThreadsPerCore selects how many SMT threads of each dedicated physical + // core become vCPUs when full-pcpus-only is set: 2 (default) exposes both + // siblings (whole-core-smt); 1 parks the sibling for isolation + // (one-per-core). Not a K8s term -- K8s has no single option for this. + ThreadsPerCore int `json:"threads_per_core,omitempty"` + // NUMATopologyPolicy mirrors the K8s Topology Manager: + // "single-numa-node"/"restricted" (strict), "best-effort" (default), "none". + NUMATopologyPolicy string `json:"numa_topology_policy"` + // IOPlacement is EVE-specific (not a K8s CPU-manager concept): "dedicated" + // (default) keeps the QEMU main-loop + iothread on the VM's dedicated cores; + // "housekeeping" pins them to the shared non-VM pool (off the hot vCPU cores). + IOPlacement string `json:"io_placement"` +} + +// PinningConfig is the on-disk structure keyed by VM UUID. +type PinningConfig struct { + Comment string `json:"_comment,omitempty"` + Domains map[string]*PinningEntry `json:"domains"` +} + +func newEmptyPinningConfig() *PinningConfig { + return &PinningConfig{ + Comment: "EVE CPU policy, aligned with Kubernetes CPUManager/Topology Manager. Keys are VM UUIDs. " + + "cpu_policy: none|static; policy_options.full-pcpus-only reserves whole physical cores; " + + "threads_per_core: 2=both SMT siblings (whole core), 1=park sibling for isolation; " + + "numa_topology_policy: single-numa-node|restricted|best-effort|none; io_placement: dedicated|housekeeping.", + Domains: map[string]*PinningEntry{}, + } +} + +func loadPinningConfig() (*PinningConfig, error) { + data, err := os.ReadFile(pinConfigFile) + if err != nil { + if os.IsNotExist(err) { + return newEmptyPinningConfig(), nil + } + return nil, err + } + cfg := newEmptyPinningConfig() + if err := json.Unmarshal(data, cfg); err != nil { + return nil, err + } + if cfg.Domains == nil { + cfg.Domains = map[string]*PinningEntry{} + } + return cfg, nil +} + +func savePinningConfig(cfg *PinningConfig) error { + if err := os.MkdirAll(pinConfigDir, 0755); err != nil { + return err + } + data, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + tmp := pinConfigFile + ".tmp" + if err := os.WriteFile(tmp, data, 0644); err != nil { + return err + } + return os.Rename(tmp, pinConfigFile) +} + +// ensureDomainInPinningConfig adds an entry for the domain if none exists, +// describing what the controller asked for so an operator has something to edit. +// Existing entries are never overwritten, so operator edits survive reboots and +// app restarts. +// +// A file that cannot be parsed is left strictly alone. This is an +// operator-editable file, so a syntax error is its likeliest failure mode, and +// rewriting it from scratch would delete every other workload's policy -- the +// one piece of evidence needed to see what went wrong, and the reason those +// workloads would silently revert to legacy pinning on their next start. +func ensureDomainInPinningConfig(config types.DomainConfig) { + uuidStr := config.UUIDandVersion.UUID.String() + cfg, err := loadPinningConfig() + if err != nil { + log.Errorf("ensureDomainInPinningConfig: %s is unreadable (%v); "+ + "leaving it untouched. No entry is added for %s.", + pinConfigFile, err, config.DisplayName) + return + } + if _, exists := cfg.Domains[uuidStr]; exists { + return + } + entry := &PinningEntry{ + DisplayName: config.DisplayName, + UUID: uuidStr, + VCpus: config.VCpus, + NUMATopologyPolicy: "best-effort", + IOPlacement: "dedicated", + } + // Conservative default that preserves today's behavior: a pinned VM gets + // static (exclusive) but topology-blind allocation -- no full-pcpus-only, + // so it maps to the legacy shared-pool pinning; a non-pinned VM is "none". + if config.VmConfig.CPUsPinned { + entry.CPUPolicy = "static" + } else { + entry.CPUPolicy = "none" + } + cfg.Domains[uuidStr] = entry + if err := savePinningConfig(cfg); err != nil { + log.Errorf("ensureDomainInPinningConfig: save failed: %v", err) + } +} + +// lookupPinningPolicy returns the topology-pinning policy an operator wrote for +// a domain. +// +// found is false when the override expresses no topology preference: no entry at +// all, cpu_policy "none", or static without full-pcpus-only. It does not mean +// "not pinned" -- whether the workload gets CPUs of its own is then decided by +// the controller's CPUsPinned flag. +// +// An unreadable or invalid file is an error rather than a silent "no +// preference", so a workload asking for whole cores fails closed instead of +// booting on shared, thread-granular CPUs while the operator believes their +// override is in force. +func lookupPinningPolicy(id uuid.UUID) (cpuallocator.PinMode, cpuallocator.NUMAPolicy, bool, error) { + cfg, err := loadPinningConfig() + if err != nil { + return cpuallocator.ModeShared, cpuallocator.NUMABestEffort, false, err + } + return pinningPolicyOf(cfg, id) +} + +func pinningPolicyOf(cfg *PinningConfig, id uuid.UUID) (cpuallocator.PinMode, + cpuallocator.NUMAPolicy, bool, error) { + entry, ok := cfg.Domains[id.String()] + if !ok { + return cpuallocator.ModeShared, cpuallocator.NUMABestEffort, false, nil + } + mode, found, err := mapPinMode(entry) + if err != nil { + return cpuallocator.ModeShared, cpuallocator.NUMABestEffort, false, err + } + return mode, mapNUMAPolicy(entry.NUMATopologyPolicy), found, nil +} + +// mapPinMode derives the allocator PinMode from a K8s-aligned entry. Only +// static + full-pcpus-only yields a topology-aware mode (found=true); "none" +// and static-without-full-pcpus-only express no topology preference. +// +// threads_per_core is validated rather than rounded off: an operator who wrote 4 +// asked for something this device does not do, and quietly giving them 2 hands +// back a workload placed differently from what the file says. +func mapPinMode(e *PinningEntry) (cpuallocator.PinMode, bool, error) { + fullPCPUsOnly := e.PolicyOptions != nil && e.PolicyOptions.FullPCPUsOnly + if e.CPUPolicy != "static" || !fullPCPUsOnly { + return cpuallocator.ModeShared, false, nil + } + switch e.ThreadsPerCore { + case 0, 2: + return cpuallocator.ModeWholeCoreSMT, true, nil + case 1: + return cpuallocator.ModeOnePerCore, true, nil + } + return cpuallocator.ModeShared, false, fmt.Errorf( + "%s: threads_per_core must be 1 or 2 (or omitted), got %d for %s", + pinConfigFile, e.ThreadsPerCore, e.DisplayName) +} + +func mapNUMAPolicy(s string) cpuallocator.NUMAPolicy { + switch s { + case "single-numa-node", "restricted": + return cpuallocator.NUMALocal + case "none": + return cpuallocator.NUMAAllowCross + default: // "best-effort" or unset + return cpuallocator.NUMABestEffort + } +} + +// lookupIOPlacement returns "housekeeping" or "dedicated" (default) for a VM. +func lookupIOPlacement(id uuid.UUID) string { + cfg, err := loadPinningConfig() + if err != nil { + log.Errorf("lookupIOPlacement: %s is unreadable (%v); assuming dedicated", + pinConfigFile, err) + return "dedicated" + } + return ioPlacementOf(cfg, id) +} + +func ioPlacementOf(cfg *PinningConfig, id uuid.UUID) string { + if e, ok := cfg.Domains[id.String()]; ok && e.IOPlacement == "housekeeping" { + return "housekeeping" + } + return "dedicated" +} diff --git a/pkg/pillar/cmd/domainmgr/pinningconfig_test.go b/pkg/pillar/cmd/domainmgr/pinningconfig_test.go new file mode 100644 index 00000000000..0f655748645 --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/pinningconfig_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +func writePinningEntryForTest(t *testing.T, id uuid.UUID, e *PinningEntry) { + t.Helper() + e.UUID = id.String() + cfg := &PinningConfig{Domains: map[string]*PinningEntry{id.String(): e}} + data, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + t.Fatalf("marshal: %v", err) + } + if err := os.WriteFile(pinConfigFile, data, 0644); err != nil { + t.Fatalf("write: %v", err) + } +} + +func fullPCPUs() *PolicyOptions { return &PolicyOptions{FullPCPUsOnly: true} } + +func TestPinningPolicy_RoundTrip(t *testing.T) { + dir := t.TempDir() + pinConfigDir = dir + pinConfigFile = filepath.Join(dir, "config.json") + + id := uuid.NewV5(uuid.NamespaceOID, "vm1") + var cfg types.DomainConfig + cfg.DisplayName = "vm1" + cfg.UUIDandVersion.UUID = id + cfg.VmConfig.VCpus = 4 + + // Non-pinned VM => default cpu_policy "none" => legacy (found=false). + ensureDomainInPinningConfig(cfg) + if _, _, found, _ := lookupPinningPolicy(id); found { + t.Fatalf("default 'none' entry => policy found must be false") + } + + // whole-core-smt = static + full-pcpus-only, threads_per_core default (2), + // strict NUMA. + writePinningEntryForTest(t, id, &PinningEntry{ + CPUPolicy: "static", + PolicyOptions: fullPCPUs(), + NUMATopologyPolicy: "single-numa-node", + }) + if mode, numa, found, _ := lookupPinningPolicy(id); !found || + mode != cpuallocator.ModeWholeCoreSMT || numa != cpuallocator.NUMALocal { + t.Fatalf("want whole-core-smt/single-numa-node/found, got mode=%v numa=%v found=%v", mode, numa, found) + } + + // one-per-core = static + full-pcpus-only + threads_per_core 1, numa none. + writePinningEntryForTest(t, id, &PinningEntry{ + CPUPolicy: "static", + PolicyOptions: fullPCPUs(), + ThreadsPerCore: 1, + NUMATopologyPolicy: "none", + }) + if mode, numa, found, _ := lookupPinningPolicy(id); !found || + mode != cpuallocator.ModeOnePerCore || numa != cpuallocator.NUMAAllowCross { + t.Fatalf("want one-per-core/none/found, got mode=%v numa=%v found=%v", mode, numa, found) + } +} + +// static without full-pcpus-only, and cpu_policy none, both fall back to legacy +// exclusive pinning (found=false). +func TestPinningPolicy_LegacyMappings(t *testing.T) { + dir := t.TempDir() + pinConfigDir = dir + pinConfigFile = filepath.Join(dir, "config.json") + id := uuid.NewV5(uuid.NamespaceOID, "legacy") + + writePinningEntryForTest(t, id, &PinningEntry{CPUPolicy: "static"}) // no full-pcpus-only + if _, _, found, _ := lookupPinningPolicy(id); found { + t.Fatalf("static without full-pcpus-only must be legacy (found=false)") + } + writePinningEntryForTest(t, id, &PinningEntry{CPUPolicy: "none", PolicyOptions: fullPCPUs()}) + if _, _, found, _ := lookupPinningPolicy(id); found { + t.Fatalf("cpu_policy none must be legacy (found=false) regardless of options") + } +} + +// Unset numa_topology_policy defaults to best-effort. +func TestPinningPolicy_NUMADefaultBestEffort(t *testing.T) { + dir := t.TempDir() + pinConfigDir = dir + pinConfigFile = filepath.Join(dir, "config.json") + id := uuid.NewV5(uuid.NamespaceOID, "be") + writePinningEntryForTest(t, id, &PinningEntry{CPUPolicy: "static", PolicyOptions: fullPCPUs()}) + if _, numa, _, _ := lookupPinningPolicy(id); numa != cpuallocator.NUMABestEffort { + t.Fatalf("unset numa_topology_policy must default to best-effort, got %v", numa) + } +} + +// A pinned VM's default entry is static-but-not-full-pcpus-only, i.e. legacy +// exclusive pinning (found=false) -- preserving today's behavior. +func TestPinningPolicy_PinnedDefaultIsLegacy(t *testing.T) { + dir := t.TempDir() + pinConfigDir = dir + pinConfigFile = filepath.Join(dir, "config.json") + id := uuid.NewV5(uuid.NamespaceOID, "pinned") + var cfg types.DomainConfig + cfg.DisplayName = "pinned" + cfg.UUIDandVersion.UUID = id + cfg.VmConfig.VCpus = 2 + cfg.VmConfig.CPUsPinned = true + ensureDomainInPinningConfig(cfg) + if _, _, found, _ := lookupPinningPolicy(id); found { + t.Fatalf("pinned VM default must be legacy (found=false)") + } +} + +func TestPinningPolicy_AbsentIsLegacy(t *testing.T) { + dir := t.TempDir() + pinConfigDir = dir + pinConfigFile = filepath.Join(dir, "config.json") + if _, _, found, _ := lookupPinningPolicy(uuid.NewV5(uuid.NamespaceOID, "none")); found { + t.Fatalf("absent VM must be legacy (found=false)") + } +} + +func TestIOPlacement(t *testing.T) { + dir := t.TempDir() + pinConfigDir = dir + pinConfigFile = filepath.Join(dir, "config.json") + + id := uuid.NewV5(uuid.NamespaceOID, "iovm") + // absent entry => default dedicated + if got := lookupIOPlacement(id); got != "dedicated" { + t.Fatalf("absent => dedicated, got %q", got) + } + writePinningEntryForTest(t, id, &PinningEntry{ + CPUPolicy: "static", PolicyOptions: fullPCPUs(), IOPlacement: "housekeeping", + }) + if got := lookupIOPlacement(id); got != "housekeeping" { + t.Fatalf("explicit housekeeping, got %q", got) + } + writePinningEntryForTest(t, id, &PinningEntry{ + CPUPolicy: "static", PolicyOptions: fullPCPUs(), IOPlacement: "", + }) + if got := lookupIOPlacement(id); got != "dedicated" { + t.Fatalf("empty io_placement => dedicated, got %q", got) + } +} diff --git a/pkg/pillar/cmd/domainmgr/placementfixes_test.go b/pkg/pillar/cmd/domainmgr/placementfixes_test.go new file mode 100644 index 00000000000..06aff61902f --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/placementfixes_test.go @@ -0,0 +1,286 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "errors" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/cputopology" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +// The placement fields are independent on the wire, so a controller can set +// isolation_tier or full_pcpus_only while leaving cpu_policy unset. Keying the +// resolution off cpu_policy alone dropped every other field without a word -- +// including the two refusals that exist precisely so a workload never runs +// believing it got a guarantee it did not. +func TestPlacementFor_PolicyWithoutCPUPolicyIsNotIgnored(t *testing.T) { + isolatePinningOverride(t) + + tests := []struct { + name string + policy types.CPUPlacementPolicy + code string + }{ + { + name: "hard isolation tier", + policy: types.CPUPlacementPolicy{ + IsolationTier: types.CPUIsolationTierHard, + }, + code: types.ErrorCodeCPUIsolationTierUnavailable, + }, + { + name: "protect disruption policy", + policy: types.CPUPlacementPolicy{ + DisruptionPolicy: types.CPUDisruptionPolicyProtect, + }, + code: types.ErrorCodeCPUPolicyInvalid, + }, + { + name: "whole cores without a dedicated policy", + policy: types.CPUPlacementPolicy{ + FullPCPUsOnly: true, ThreadsPerCore: 1, + }, + code: types.ErrorCodeCPUPolicyInvalid, + }, + { + name: "single NUMA node without a dedicated policy", + policy: types.CPUPlacementPolicy{ + NUMAPolicy: types.CPUNUMAPolicySingleNode, + }, + code: types.ErrorCodeCPUPolicyInvalid, + }, + { + name: "threads per core out of range", + policy: types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + ThreadsPerCore: 4, + }, + code: types.ErrorCodeCPUPolicyInvalid, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := pinnedConfigForTest("policy-"+tt.name, tt.policy) + _, err := placementFor(&config) + var perr *placementError + if !errors.As(err, &perr) { + t.Fatalf("want a placement error, got %v", err) + } + if perr.Code != tt.code { + t.Errorf("want code %q, got %q (%v)", tt.code, perr.Code, err) + } + }) + } +} + +// A shared policy with no other field set is a complete, satisfiable intent: the +// workload runs unpinned. Rejecting it would refuse the one policy that asks for +// nothing. +func TestPlacementFor_SharedPolicyIsAccepted(t *testing.T) { + isolatePinningOverride(t) + config := pinnedConfigForTest("shared", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyShared, + }) + placement, err := placementFor(&config) + if err != nil { + t.Fatalf("a shared policy must be accepted: %v", err) + } + if placement.TopologyAware { + t.Error("a shared policy is not topology-aware") + } +} + +// The plan ranks thread-granular workloads last but does account for them. +// Allocating one on arrival instead of taking its planned CPUs hands it the +// lowest-numbered free threads -- exactly the cores the plan set aside whole for +// a whole-core workload -- so which of the two started first decided which one +// failed. +func TestAllocateCPUs_LegacyPinningTakesItsPlannedCPUs(t *testing.T) { + isolatePinningOverride(t) + ps := testPubSub(t) + + // Two vCPUs each: a whole-core workload needs one full core, the legacy one + // takes two individual threads. + wholeCore := pinnedConfigForTest("wholecore", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }) + legacy := pinnedConfigForTest("legacy", types.CPUPlacementPolicy{}) + + ctx := &domainContext{ + placer: testPlacer(t), + cpuTopologyPinningSupported: true, + cpuPinningSupported: true, + pubDomainStatus: testPublication(t, ps, types.DomainStatus{}), + subDomainConfig: testDomainConfigSub(t, ps, wholeCore, legacy), + } + + // The legacy workload activates first, when nothing is running. + var legacyStatus types.DomainStatus + if err := assignCPUs(ctx, &legacy, &legacyStatus); err != nil { + t.Fatalf("legacy placement failed: %v", err) + } + var wholeCoreStatus types.DomainStatus + if err := assignCPUs(ctx, &wholeCore, &wholeCoreStatus); err != nil { + t.Fatalf("whole-core placement failed after the legacy one: %v", err) + } + + planned := claimedPlan(ctx) + want := planned[legacy.UUIDandVersion.UUID] + if want == nil { + t.Fatal("the plan says nothing about the legacy workload") + } + for _, cpu := range legacyStatus.VmConfig.CPUs { + if !containsLCPU(want.OrderedHostCPUs, cpu) { + t.Errorf("legacy workload got CPU %d, which the plan gave to someone "+ + "else (planned %v, whole-core got %v)", cpu, want.OrderedHostCPUs, + wholeCoreStatus.VmConfig.CPUs) + } + } + for _, cpu := range wholeCoreStatus.OrderedCPUs { + if containsUint32(legacyStatus.VmConfig.CPUs, cpu) { + t.Errorf("whole-core workload and legacy workload share CPU %d", cpu) + } + } +} + +// A plan entry sized for a different vCPU count must not be applied: the guest +// is launched with an -smp topology derived from the assignment, so a mismatch +// makes QEMU refuse to start, and the retry would reuse the same stale +// assignment forever. +func TestClaimPlannedPlacement_RejectsAStaleVCPUCount(t *testing.T) { + ctx := &domainContext{placer: testPlacer(t)} + id := uuid.NewV5(uuid.NamespaceOID, "resized") + plan := map[uuid.UUID]cpuallocator.Result{id: plannedOn(0, 4)} + + if got := claimPlannedPlacement(ctx, id, 2, plan); got == nil { + t.Fatal("a plan for the configured vCPU count must be claimable") + } + if got := claimPlannedPlacement(ctx, id, 4, plan); got != nil { + t.Errorf("a plan for 2 vCPUs must not be claimed for 4, got %v", + got.OrderedHostCPUs) + } +} + +// A parked SMT sibling is consumed by the workload that parked it. A plan slot +// whose parked CPU is held by someone else therefore cannot be taken. +func TestClaimPlannedPlacement_ParkedCPUHeldBlocksTheClaim(t *testing.T) { + placer := testPlacer(t) + ctx := &domainContext{placer: placer} + other := uuid.NewV5(uuid.NamespaceOID, "other") + if err := placer.Reserve(other, []uint32{4}); err != nil { + t.Fatalf("Reserve: %v", err) + } + + id := uuid.NewV5(uuid.NamespaceOID, "onepercore") + plan := map[uuid.UUID]cpuallocator.Result{id: { + Status: cpuallocator.Success, + Assignment: &cpuallocator.Assignment{ + OrderedHostCPUs: []cputopology.LCPU{0}, + ParkedCPUs: []cputopology.LCPU{4}, + }, + }} + if got := claimPlannedPlacement(ctx, id, 1, plan); got != nil { + t.Errorf("the claim must fail while CPU 4 is held elsewhere, got %v", + got.OrderedHostCPUs) + } +} + +// The housekeeping IO set is chosen when the workload activates and never +// revisited, so it has to be drawn from CPUs no workload can ever be given. +// Anything else is invalidated by the next deployment. +func TestEmulatorHousekeepingCPUs_PrefersTheCPUsReservedForEVE(t *testing.T) { + ctx := &domainContext{placer: testPlacer(t), cpusReserved: 2} + got := emulatorHousekeepingCPUs(ctx, nil) + if len(got) != 2 || got[0] != 0 || got[1] != 1 { + t.Errorf("want the reserved CPUs [0 1], got %v", got) + } +} + +// A synthesized topology cannot say which CPUs are SMT siblings, so a whole-core +// request placed against it would park nothing and share cores while reporting +// an optimal placement. +func TestAssignCPUs_WholeCoreRefusedOnDegradedTopology(t *testing.T) { + isolatePinningOverride(t) + ctx := &domainContext{ + placer: testPlacer(t), + cpuTopologyPinningSupported: true, + cpuTopologyDegraded: true, + } + config := pinnedConfigForTest("wholecore", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }) + var status types.DomainStatus + + err := assignCPUs(ctx, &config, &status) + var perr *placementError + if !errors.As(err, &perr) || perr.Code != types.ErrorCodeCPUTopologyUnsupported { + t.Fatalf("want %q, got %v", types.ErrorCodeCPUTopologyUnsupported, err) + } + if len(status.VmConfig.CPUs) != 0 { + t.Errorf("a refused placement must reserve nothing, got %v", + status.VmConfig.CPUs) + } +} + +// releaseCPUs is the single release path, so it must leave no pin state behind. +// A stale CPUsPinned with the housekeeping set in VmConfig.CPUs is read by the +// post-restart reseed as "these are exclusively mine", reserving most of the +// node to one workload. +func TestReleaseCPUs_ClearsThePinnedFlag(t *testing.T) { + isolatePinningOverride(t) + ps := testPubSub(t) + ctx := &domainContext{ + placer: testPlacer(t), + pubCPUPoolStatus: testPublication(t, ps, types.CPUPoolStatus{}), + } + config := pinnedConfigForTest("released", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, + }) + var status types.DomainStatus + status.UUIDandVersion = config.UUIDandVersion + if err := assignCPUs(ctx, &config, &status); err != nil { + t.Fatalf("placement failed: %v", err) + } + if !status.VmConfig.CPUsPinned { + t.Fatal("precondition: the workload should be marked pinned") + } + + releaseCPUs(ctx, &status) + if status.VmConfig.CPUsPinned { + t.Error("CPUsPinned must be cleared with the rest of the pin state") + } +} + +func claimedPlan(ctx *domainContext) map[uuid.UUID]*cpuallocator.Assignment { + out := map[uuid.UUID]*cpuallocator.Assignment{} + for id, result := range planPinnedPlacement(ctx) { + if result.Status == cpuallocator.Success { + out[id] = result.Assignment + } + } + return out +} + +func containsLCPU(cpus []cputopology.LCPU, want uint32) bool { + for _, cpu := range cpus { + if uint32(cpu) == want { + return true + } + } + return false +} + +func containsUint32(cpus []uint32, want uint32) bool { + for _, cpu := range cpus { + if cpu == want { + return true + } + } + return false +} diff --git a/pkg/pillar/cmd/domainmgr/placementpolicy.go b/pkg/pillar/cmd/domainmgr/placementpolicy.go new file mode 100644 index 00000000000..1d6107cb424 --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/placementpolicy.go @@ -0,0 +1,455 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "errors" + "fmt" + "strings" + + "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" +) + +// placementError carries a machine-parseable code alongside the message, so the +// failure reaches the controller as a structured ErrorInfo rather than prose. +type placementError struct { + Code string + Msg string + // Retry is what would change the outcome, for the failures that can say + // something more specific than their code's generic condition (which + // workloads are in the way, how many cores short the node is). Empty means + // the code's generic condition is used; it never reaches the controller + // empty, see retryConditionFor. + Retry string +} + +func (e *placementError) Error() string { return e.Msg } + +func placementErrorf(code, format string, args ...interface{}) *placementError { + return &placementError{Code: code, Msg: fmt.Sprintf(format, args...)} +} + +// retryWhen attaches the specific condition this failure clears under, for the +// cases where the generic per-code text can be improved on with real numbers or +// real workload names. +func (e *placementError) retryWhen(format string, args ...interface{}) *placementError { + e.Retry = fmt.Sprintf(format, args...) + return e +} + +// noAutoRetry is appended to every placement retry condition. +// +// A CPU-placement failure is not re-attempted by the device: the workload holds +// no CPUs, sets neither BootFailed nor ConfigFailed, and stays failed until the +// controller or the operator acts -- exactly how EVE already treats a workload +// that cannot get an assigned PCI device. Saying so is the point of the field +// here: a controller told only "when cores free" would sit and wait for a +// recovery that never comes on its own. +const noAutoRetry = " The device does not re-attempt placement on its own." + +// retryConditionFor is the condition a placement failure clears under when the +// failure itself has nothing more specific to add. Every code produced on this +// path has an entry: an ErrorInfo that says "insufficient" with an empty +// retry_condition tells a controller that something is wrong and nothing about +// what would make it right, which is the one outcome worse than a blunt message. +func retryConditionFor(code string) string { + switch code { + case types.ErrorCodeCPUPlacementInsufficient: + return "Stop another pinned workload to free whole physical cores, or " + + "deploy this one on a node with more cores, then start it again." + + noAutoRetry + case types.ErrorCodeCPUPlacementNeedsRepack: + return "Restart the pinned workloads together so they repack, then start " + + "this one again." + noAutoRetry + case types.ErrorCodeCPUPolicyOddVCPU: + return "Change the workload's vCPU count to an even number, or set " + + "threads_per_core=1, and deploy it again. No change on the node makes " + + "an odd count placeable on two-thread cores." + noAutoRetry + case types.ErrorCodeCPUIsolationTierUnavailable: + return "Lower the workload's isolation_tier to soft or none, or deploy it " + + "on a node booted with CPU isolation. This node cannot change its " + + "isolation without a kernel command-line change and a reboot." + noAutoRetry + case types.ErrorCodeCPUTopologyUnsupported: + return "Deploy the workload on a node whose hypervisor can pin individual " + + "vCPUs, or turn off whole-core placement (full_pcpus_only) for it. No " + + "amount of free CPUs makes this node able to honour the request." + noAutoRetry + case types.ErrorCodeCPUPolicyInvalid: + return "Change the workload's CPU placement policy to one this node " + + "implements and deploy it again. No change on the node makes the " + + "request as written satisfiable." + noAutoRetry + } + // A code with no entry is still better served by a true statement than by + // an empty field: every failure on this path is fail-closed, so it needs + // either the workload's configuration or the node to change. + return "Change the workload's CPU placement configuration, or deploy it on a " + + "node that can satisfy it, then start it again." + noAutoRetry +} + +// resolvedPlacement is a workload's CPU placement intent translated into the +// allocator's vocabulary. +type resolvedPlacement struct { + // TopologyAware selects the whole-physical-core, SMT/NUMA-aware allocator. + // When false the workload is either not pinned at all or pinned at + // thread granularity through the legacy shared-pool path. + TopologyAware bool + Mode cpuallocator.PinMode + NUMA cpuallocator.NUMAPolicy + // IOHousekeeping pins the emulator/IO threads onto the housekeeping set + // instead of leaving them on the workload's dedicated cores. + IOHousekeeping bool +} + +// resolvePlacement translates a placement intent into allocator parameters. It +// rejects intents this device cannot satisfy rather than silently downgrading +// them, so a workload never runs believing it got guarantees it did not. +func resolvePlacement(p types.CPUPlacementPolicy) (resolvedPlacement, error) { + if err := rejectUnenforceableFields(p); err != nil { + return resolvedPlacement{}, err + } + if !p.IsolationTier.SupportedBySoftIsolation() { + return resolvedPlacement{}, placementErrorf( + types.ErrorCodeCPUIsolationTierUnavailable, + "isolation tier %q requires a kernel command-line change and is not supported on this node", + p.IsolationTier) + } + + // Nothing on the device defers a node-level disruptive action yet, so + // accepting "protect" would tell the controller its workload is shielded + // while a reboot or an upgrade still takes it down unannounced. Refuse it + // for the same reason the hard isolation tier is refused: an unenforced + // guarantee is worse than a rejected one. + if p.DisruptionPolicy == types.CPUDisruptionPolicyProtect { + return resolvedPlacement{}, placementErrorf(types.ErrorCodeCPUPolicyInvalid, + "disruption policy %q is not implemented on this node; "+ + "a node-level action can still take this workload down", + p.DisruptionPolicy) + } + + res := resolvedPlacement{ + TopologyAware: p.IsTopologyAware(), + NUMA: numaPolicyFor(p.NUMAPolicy), + IOHousekeeping: p.IOPlacement == types.CPUIOPlacementHousekeeping, + } + if !res.TopologyAware { + res.Mode = cpuallocator.ModeShared + return res, nil + } + switch tpc := p.EffectiveThreadsPerCore(); tpc { + case 1: + res.Mode = cpuallocator.ModeOnePerCore + case 2: + res.Mode = cpuallocator.ModeWholeCoreSMT + default: + return resolvedPlacement{}, placementErrorf(types.ErrorCodeCPUPolicyInvalid, + "threads_per_core must be 1 or 2, got %d", tpc) + } + return res, nil +} + +// rejectUnenforceableFields refuses a policy whose whole-core fields cannot take +// effect as written. +// +// full_pcpus_only, threads_per_core and numa_policy only mean anything for a +// workload that gets CPUs of its own, so a policy that sets them without +// cpu_policy=dedicated describes a placement this device will not perform. The +// request is refused rather than honoured in part, for the same reason the hard +// isolation tier is: the workload would otherwise run on shared, thread-granular +// CPUs while the controller believes it asked for whole cores on one NUMA node. +// +// threads_per_core is checked here as well as on the whole-core path, so an +// out-of-range value is reported as the invalid policy it is instead of being +// quietly dropped along with the rest. +func rejectUnenforceableFields(p types.CPUPlacementPolicy) error { + if p.IsDedicated() { + return nil + } + var set []string + if p.FullPCPUsOnly { + set = append(set, "full_pcpus_only") + } + if p.ThreadsPerCore != 0 { + set = append(set, fmt.Sprintf("threads_per_core=%d", p.ThreadsPerCore)) + } + if p.NUMAPolicy != types.CPUNUMAPolicyUnspecified { + set = append(set, fmt.Sprintf("numa_policy=%s", p.NUMAPolicy)) + } + if len(set) == 0 { + return nil + } + return placementErrorf(types.ErrorCodeCPUPolicyInvalid, + "%s require cpu_policy=dedicated, which this workload does not request "+ + "(cpu_policy=%s)", strings.Join(set, ", "), p.Policy) +} + +func numaPolicyFor(p types.CPUNUMAPolicy) cpuallocator.NUMAPolicy { + switch p { + case types.CPUNUMAPolicyNone: + return cpuallocator.NUMAAllowCross + // Both mean "one node or fail" for now. They differ in Kubernetes only by + // how many *other* aligned resources are weighed, and CPU placement weighs + // none yet -- see the comment on CPUNUMAPolicyRestricted. + case types.CPUNUMAPolicyRestricted, types.CPUNUMAPolicySingleNode: + return cpuallocator.NUMALocal + default: // unspecified or best-effort + return cpuallocator.NUMABestEffort + } +} + +// placementErrorDescription turns a placement failure into a status error, +// carrying the structured code when the failure has one so the controller can +// distinguish "a repack would fix this" from "nothing would" without parsing the +// message text, plus the retry condition, which is the only place the report says +// what would change the answer. +// +// The message says what happened; the retry condition says what to do about it. +// They are deliberately not the same sentence: a UI shows the two fields in +// different places, and repeating the message in the condition wastes the only +// field an operator can act on. +func placementErrorDescription(err error) types.ErrorDescription { + description := types.ErrorDescription{Error: err.Error()} + var placementErr *placementError + if errors.As(err, &placementErr) { + description.ErrorCode = placementErr.Code + description.ErrorRetryCondition = placementErr.Retry + if description.ErrorRetryCondition == "" { + description.ErrorRetryCondition = retryConditionFor(placementErr.Code) + } + } + return description +} + +// placementErrorCode maps an allocator outcome onto the published error-code +// registry. NeedsRebalance and Insufficient are deliberately distinct: the +// first says a repack would fix this, the second that nothing would. +func placementErrorCode(s cpuallocator.Status) string { + switch s { + case cpuallocator.NeedsRebalance: + return types.ErrorCodeCPUPlacementNeedsRepack + case cpuallocator.Insufficient: + return types.ErrorCodeCPUPlacementInsufficient + default: + return types.ErrorCodeCPUPolicyInvalid + } +} + +// liveAllocationError classifies a failed live allocation by consulting the +// plan, and is the only place the two placement failure codes are chosen +// between. +// +// The live allocator can only say "this does not fit among what is free right +// now". The plan is computed from an empty slate over the whole configured set, +// so it answers the question the controller actually needs answered: whether the +// workload fits on this node at all. Without that distinction every fragmented +// node reports cpu.placement.insufficient, and the controller -- whose only +// remedy is restarting the pinned workloads so they repack -- cannot tell a node +// it can fix from one it cannot, so it never tries. +// +// Only a shortage is reinterpreted. InvalidRequest means the caller asked for +// something impossible, which is a bug on this side; blaming it on the running +// workloads would send the operator repacking a node that is not the problem. +// +// blockers names the workloads standing on the CPUs the plan set aside for this +// one, when they are known; an empty list only costs the retry condition its +// names, never its meaning. +func liveAllocationError(displayName string, id uuid.UUID, live cpuallocator.Result, + plan map[uuid.UUID]cpuallocator.Result, blockers []string) *placementError { + switch live.Status { + case cpuallocator.Insufficient, cpuallocator.NeedsRebalance: + if planned, isPlanned := plan[id]; isPlanned && + planned.Status == cpuallocator.Success && planned.Assignment != nil { + return placementErrorf(types.ErrorCodeCPUPlacementNeedsRepack, + "topology pinning for %s: %s; the planned placement on host CPUs %v "+ + "does fit this node", + displayName, live.Message, assignmentCPUs(planned.Assignment)). + retryWhen("%sRestarting the pinned workloads together so they repack "+ + "lets this one take its planned CPUs.%s", + heldByClause(blockers), noAutoRetry) + } + } + err := placementErrorf(placementErrorCode(live.Status), + "topology pinning for %s: %s", displayName, live.Message) + if condition := shortageRetryCondition(live); condition != "" { + return err.retryWhen("%s", condition) + } + return err +} + +// heldByClause names the workloads in the way, as a sentence the retry condition +// can lead with. Empty when nothing is known, so the condition still reads. +func heldByClause(blockers []string) string { + if len(blockers) == 0 { + return "" + } + return fmt.Sprintf("The planned CPUs are held by %s. ", + strings.Join(blockers, ", ")) +} + +// shortageRetryCondition states in numbers the condition a shortage clears +// under, which is what a controller needs in order to decide between waiting for +// capacity and placing the workload somewhere else. +// +// It returns "" when the allocator counted no cores -- a thread-granular request +// is not a whole-core shortage -- and the code's generic condition is used +// instead. It deliberately does not claim that no arrangement could ever fit: +// this path is also reached for a workload the plan says nothing about, where +// that is not known. +func shortageRetryCondition(res cpuallocator.Result) string { + if res.CoresNeeded <= 0 { + return "" + } + switch res.Status { + case cpuallocator.NeedsRebalance: + return fmt.Sprintf("It can run once %d whole physical cores are free within a "+ + "single NUMA node (%d free across all nodes now) and the workload is "+ + "started again; restarting the pinned workloads together so they repack "+ + "may also achieve it.%s", res.CoresNeeded, res.CoresFree, noAutoRetry) + case cpuallocator.Insufficient: + return fmt.Sprintf("It can run once %d whole physical cores are free at once "+ + "(%d free now) and the workload is started again; freeing them needs "+ + "another pinned workload to stop, or a node with more cores.%s", + res.CoresNeeded, res.CoresFree, noAutoRetry) + } + return "" +} + +// validateVCPUCount is the device-side backstop for constraints the controller +// is also expected to enforce at deploy time. +func validateVCPUCount(r resolvedPlacement, vcpus int) error { + if r.TopologyAware && r.Mode == cpuallocator.ModeWholeCoreSMT && vcpus%2 != 0 { + return placementErrorf(types.ErrorCodeCPUPolicyOddVCPU, + "whole-core-smt requires an even vCPU count, got %d", vcpus) + } + return nil +} + +// cpuIntent is everything a placement decision needs to know about a workload. +// +// It exists because the decision has to be made for workloads that have no +// DomainConfig yet: the plan covers every app the controller intends to run, +// and most of them have not got as far as a DomainConfig when the first one is +// placed. Both sources -- a DomainConfig and an entry in the demand set -- +// reduce to this. +type cpuIntent struct { + id uuid.UUID + displayName string + vcpus int + // pinned is the controller's legacy pin flag. It is not the final answer: + // see cpuIntentPinned. + pinned bool + policy types.CPUPlacementPolicy +} + +func intentOfConfig(config *types.DomainConfig) cpuIntent { + return cpuIntent{ + id: config.UUIDandVersion.UUID, + displayName: config.DisplayName, + vcpus: config.VCpus, + pinned: config.VmConfig.CPUsPinned, + policy: config.VmConfig.CPUPlacement, + } +} + +func intentOfDemand(app types.AppCPUDemand) cpuIntent { + return cpuIntent{ + id: app.UUID, + displayName: app.DisplayName, + vcpus: app.VCpus, + pinned: app.CPUsPinned, + policy: app.CPUPlacement, + } +} + +// placementFor resolves the effective placement for a domain. +// +// The controller's intent is authoritative whenever it sent one. The +// operator-editable /persist override only applies to workloads the controller +// said nothing about, which keeps it useful for bring-up and manual testing on +// a device with no policy-aware controller, without letting a stale local file +// silently contradict what the controller asked for. +func placementFor(config *types.DomainConfig) (resolvedPlacement, error) { + return placementForIntent(intentOfConfig(config)) +} + +func placementForIntent(intent cpuIntent) (resolvedPlacement, error) { + if controllerSentPolicy(intent.policy) { + return resolvePlacement(intent.policy) + } + return placementFromPersist(intent.id) +} + +// controllerSentPolicy reports whether the controller expressed any CPU +// placement intent at all. +// +// It cannot key off cpu_policy alone. The placement fields are independent on +// the wire, so a controller can set isolation_tier or full_pcpus_only while +// leaving cpu_policy unset -- and gating on cpu_policy meant every other field +// was dropped without a word, taking the two refusals that exist to prevent an +// unenforced guarantee (hard isolation, protect disruption) with it. +// +// The zero value still means "no policy", which is what keeps the /persist +// override available for a workload the controller said nothing about. +func controllerSentPolicy(p types.CPUPlacementPolicy) bool { + return p != types.CPUPlacementPolicy{} +} + +// effectiveCPUsPinned reports whether a workload must get host CPUs of its own. +// +// It exists because the two sources of placement intent disagree about how +// pinning is switched on. The controller sets CPUsPinned (zedagent derives it +// from a dedicated policy), but the /persist override cannot: it is an operator +// file, not part of the app config. Without this, asking for +// static + full-pcpus-only in /persist did nothing at all unless the controller +// also happened to pin the workload -- which defeats the point of the override, +// namely bringing whole-core placement up on a device whose controller knows +// nothing about it. +// +// Precedence is unchanged: the controller's intent wins whenever it sent any, +// including an explicit "shared", and the override only speaks for workloads +// the controller said nothing about. +func effectiveCPUsPinned(config *types.DomainConfig) bool { + return cpuIntentPinned(intentOfConfig(config)) +} + +func cpuIntentPinned(intent cpuIntent) bool { + if intent.pinned { + return true + } + if controllerSentPolicy(intent.policy) { + return false + } + placement, err := placementFromPersist(intent.id) + return err == nil && placement.TopologyAware +} + +// placementFromPersist reads the operator-editable /persist/pinning override. +// +// An unreadable or invalid file leaves the workload on the controller's intent +// alone, logged at error severity. It cannot fail the workload instead: the +// override speaks only for workloads the controller said nothing about, so +// refusing to start would take down workloads that never asked for anything the +// file could grant. What it must not do is quietly pretend the file granted +// something -- hence TopologyAware stays false, and no placement quality is +// claimed. +func placementFromPersist(id uuid.UUID) (resolvedPlacement, error) { + cfg, err := loadPinningConfig() + if err != nil { + log.Errorf("CPU placement: %s is unreadable (%v); ignoring the operator "+ + "override for %s and using the controller's intent alone", + pinConfigFile, err, id) + return resolvedPlacement{Mode: cpuallocator.ModeShared}, nil + } + mode, numa, found, err := pinningPolicyOf(cfg, id) + if err != nil { + log.Errorf("CPU placement: %v; ignoring the operator override for %s", err, id) + return resolvedPlacement{Mode: cpuallocator.ModeShared}, nil + } + return resolvedPlacement{ + TopologyAware: found, + Mode: mode, + NUMA: numa, + IOHousekeeping: ioPlacementOf(cfg, id) == "housekeeping", + }, nil +} diff --git a/pkg/pillar/cmd/domainmgr/placementpolicy_test.go b/pkg/pillar/cmd/domainmgr/placementpolicy_test.go new file mode 100644 index 00000000000..c3b473e0744 --- /dev/null +++ b/pkg/pillar/cmd/domainmgr/placementpolicy_test.go @@ -0,0 +1,852 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package domainmgr + +import ( + "errors" + "path/filepath" + "strings" + "testing" + + "github.com/lf-edge/eve/pkg/pillar/cpuallocator" + "github.com/lf-edge/eve/pkg/pillar/cputopology" + "github.com/lf-edge/eve/pkg/pillar/pubsub" + "github.com/lf-edge/eve/pkg/pillar/types" + uuid "github.com/satori/go.uuid" + "github.com/sirupsen/logrus" +) + +// testPlacer builds a placer over a 4-core SMT2 host, enough for any placement +// these tests ask for. +func testPlacer(t *testing.T) *cpuallocator.Placer { + t.Helper() + var infos []cputopology.CoreInfo + for thread := uint(0); thread < 2; thread++ { + for core := uint(0); core < 4; core++ { + infos = append(infos, cputopology.CoreInfo{LCore: thread*4 + core, CoreID: core}) + } + } + placer, err := cpuallocator.NewPlacer(cputopology.BuildTopology(infos), 0) + if err != nil { + t.Fatalf("NewPlacer: %v", err) + } + return placer +} + +// isolatePinningOverride points the operator-editable /persist override at an +// empty temporary file, so a test neither reads the host's copy nor inherits +// one another test left behind in these package-level paths. +func isolatePinningOverride(t *testing.T) { + t.Helper() + dir := t.TempDir() + pinConfigDir = dir + pinConfigFile = filepath.Join(dir, "config.json") +} + +// testPubSub builds an in-memory pubsub for tests that need real publications. +func testPubSub(t *testing.T) *pubsub.PubSub { + t.Helper() + return pubsub.New(pubsub.NewMemoryDriver(), logrus.StandardLogger(), log) +} + +func testPublication(t *testing.T, ps *pubsub.PubSub, topic interface{}) pubsub.Publication { + t.Helper() + pub, err := ps.NewPublication(pubsub.PublicationOptions{ + AgentName: agentName, + TopicType: topic, + }) + if err != nil { + t.Fatalf("NewPublication(%T): %v", topic, err) + } + return pub +} + +// testDomainConfigSub publishes the given configs as zedmanager would and +// returns a subscription over them, so code that walks the configured set +// (placement planning, cpuset redistribution) has something to walk. +func testDomainConfigSub(t *testing.T, ps *pubsub.PubSub, + configs ...types.DomainConfig) pubsub.Subscription { + t.Helper() + // Persistent on both ends so the subscription populates from what is + // already published, the way a real subscriber picks up existing config. + pub, err := ps.NewPublication(pubsub.PublicationOptions{ + AgentName: "zedmanager", + TopicType: types.DomainConfig{}, + Persistent: true, + }) + if err != nil { + t.Fatalf("NewPublication(DomainConfig): %v", err) + } + for _, config := range configs { + if err := pub.Publish(config.Key(), config); err != nil { + t.Fatalf("Publish(DomainConfig): %v", err) + } + } + sub, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "zedmanager", + MyAgentName: agentName, + TopicImpl: types.DomainConfig{}, + Activate: true, + Persistent: true, + }) + if err != nil { + t.Fatalf("NewSubscription(DomainConfig): %v", err) + } + return sub +} + +// sharedConfigForTest is a workload with no CPU placement intent: it runs on +// whatever the pinned workloads have left over, so its cpuset is recomputed +// every time the dedicated set changes. +func sharedConfigForTest(name string) types.DomainConfig { + var config types.DomainConfig + config.UUIDandVersion.UUID = uuid.NewV5(uuid.NamespaceOID, name) + config.DisplayName = name + config.VmConfig.VCpus = 2 + return config +} + +func publishedCPUs(t *testing.T, pub pubsub.Publication, key string) []uint32 { + t.Helper() + item, err := pub.Get(key) + if err != nil { + t.Fatalf("no DomainStatus published for %s: %v", key, err) + } + return item.(types.DomainStatus).VmConfig.CPUs +} + +func pinnedConfigForTest(name string, policy types.CPUPlacementPolicy) types.DomainConfig { + var config types.DomainConfig + config.UUIDandVersion.UUID = uuid.NewV5(uuid.NamespaceOID, name) + config.DisplayName = name + config.VmConfig.VCpus = 2 + config.VmConfig.CPUsPinned = true + config.VmConfig.CPUPlacement = policy + return config +} + +// A hypervisor that cannot pin individual vCPUs must refuse a whole-core +// request outright. Every hypervisor that reports CPUPinning gets this far, but +// only kvm turns the reservation into per-vCPU pinning and a guest SMT +// topology; accepting it elsewhere reports the workload as optimally placed +// while nothing was pinned at all. +func TestAssignCPUs_WholeCoreRejectedWithoutTopologyPinning(t *testing.T) { + ctx := &domainContext{placer: testPlacer(t), cpuTopologyPinningSupported: false} + config := pinnedConfigForTest("wholecore", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }) + var status types.DomainStatus + + err := assignCPUs(ctx, &config, &status) + if err == nil { + t.Fatal("whole-core placement must be refused when it cannot be applied") + } + var perr *placementError + if !errors.As(err, &perr) || perr.Code != types.ErrorCodeCPUTopologyUnsupported { + t.Fatalf("expected %q, got %v", types.ErrorCodeCPUTopologyUnsupported, err) + } + if len(status.VmConfig.CPUs) != 0 || len(status.OrderedCPUs) != 0 { + t.Errorf("a refused placement must not reserve CPUs, got %v/%v", + status.VmConfig.CPUs, status.OrderedCPUs) + } + if status.PlacementQuality != types.CPUPlacementQualityUnspecified { + t.Errorf("a refused placement must not be reported as placed, got %v", + status.PlacementQuality) + } +} + +// Thread-granular pinning needs only a cpuset, so it must keep working on a +// hypervisor without per-vCPU pinning. +func TestAssignCPUs_ThreadGranularPinningStillWorks(t *testing.T) { + ctx := &domainContext{placer: testPlacer(t), cpuTopologyPinningSupported: false} + config := pinnedConfigForTest("threadgranular", types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, + }) + var status types.DomainStatus + + if err := assignCPUs(ctx, &config, &status); err != nil { + t.Fatalf("dedicated without full-pcpus-only must still be placed: %v", err) + } + if len(status.VmConfig.CPUs) != config.VCpus { + t.Errorf("want %d CPUs assigned, got %v", config.VCpus, status.VmConfig.CPUs) + } +} + +// startThreadGranularApp places a thread-granular dedicated workload, which is +// the only workload that can fragment the node: it takes individual threads, so +// every physical core it lands on is left half-owned and coreIsDedicated then +// refuses that core to any whole-core request. Whole-core workloads cannot +// produce this state -- they hand back whole cores when they stop. +// startThreadGranularApp puts a thread-granular workload on the node the way it +// would have landed when it was the only workload there: lowest CPUs first, one +// thread per core. +// +// It goes through the allocator rather than through assignCPUs on purpose. A +// workload the current plan accounts for takes its planned CPUs on both the +// whole-core and the thread-granular path, so it cannot fragment the node; what +// can is a workload already running on CPUs the plan -- computed later, over a +// larger set -- would not have chosen for it. A running workload cannot be moved, +// which is exactly why the remedy is a repack and not a reshuffle. +func startThreadGranularApp(t *testing.T, ctx *domainContext, + demand types.AppCPUDemand) { + t.Helper() + cpus, err := ctx.placer.AllocateShared(demand.UUID, demand.VCpus) + if err != nil { + t.Fatalf("thread-granular placement must succeed: %v", err) + } + if len(cpus) != demand.VCpus { + t.Fatalf("want %d threads taken, got %v", demand.VCpus, cpus) + } +} + +// A workload that does not fit among the free CPUs but does fit in the plan is +// blocked only by where the running workloads sit, and restarting them to repack +// is the controller's remedy. Reporting it as "insufficient" -- as the fallback +// path did, by never consulting the plan -- hides the one situation the repack +// code exists for. +func TestAssignCPUs_RepackableShortageReportsNeedsRepack(t *testing.T) { + isolatePinningOverride(t) + cpuPlanFile = filepath.Join(t.TempDir(), "cpuplan.json") + + // Four threads on a 4-core SMT host, one per core: four threads stay idle + // but not a single whole core is left. + // + fragmenter := threadGranularDemand("thread-granular", 4) + wholeCore := wholeCoreDemand("wholecore", 2) + + ps := testPubSub(t) + ctx := &domainContext{ + placer: testPlacer(t), + subCPUDemandSet: testCPUDemandSub(t, ps, fragmenter, wholeCore), + subDomainConfig: testDomainConfigSub(t, ps), + cpuTopologyPinningSupported: true, + } + startThreadGranularApp(t, ctx, fragmenter) + + config := pinnedConfigForTest(wholeCore.DisplayName, wholeCore.CPUPlacement) + var status types.DomainStatus + status.UUIDandVersion = config.UUIDandVersion + + err := assignCPUs(ctx, &config, &status) + if err == nil { + t.Fatalf("no whole core is free, placement must fail; got cpus=%v", + status.VmConfig.CPUs) + } + var perr *placementError + if !errors.As(err, &perr) || perr.Code != types.ErrorCodeCPUPlacementNeedsRepack { + t.Fatalf("a workload that the plan does place must be reported as "+ + "repackable, want %q, got %v", types.ErrorCodeCPUPlacementNeedsRepack, err) + } + // The operator has to be able to see what a repack would achieve. + if !strings.Contains(perr.Msg, "[0 4]") { + t.Errorf("the message must name the planned CPUs, got %q", perr.Msg) + } + // And who is standing on them: nothing restarts workloads by itself, so + // "a repack would help" is only actionable with a name attached. + condition := placementErrorDescription(err).ErrorRetryCondition + if !strings.Contains(condition, fragmenter.DisplayName) { + t.Errorf("the retry condition must name the workload holding the planned "+ + "CPUs (%s), got %q", fragmenter.DisplayName, condition) + } + if len(status.VmConfig.CPUs) != 0 || len(status.OrderedCPUs) != 0 { + t.Errorf("a failed placement must not reserve CPUs, got %v/%v", + status.VmConfig.CPUs, status.OrderedCPUs) + } + if status.PlacementQuality != types.CPUPlacementQualityUnspecified { + t.Errorf("a workload that never started has no placement quality, got %v", + status.PlacementQuality) + } +} + +// The same fragmented node, but a request no arrangement of workloads could +// satisfy. Here "insufficient" is the honest answer, and the allocator's +// shortage explanation must survive. +func TestAssignCPUs_UnfittableRequestStaysInsufficient(t *testing.T) { + isolatePinningOverride(t) + cpuPlanFile = filepath.Join(t.TempDir(), "cpuplan.json") + + fragmenter := threadGranularDemand("thread-granular", 4) + // Five whole cores on a four-core host: unplaceable even on an empty node. + tooBig := wholeCoreDemand("toobig", 10) + + ps := testPubSub(t) + ctx := &domainContext{ + placer: testPlacer(t), + subCPUDemandSet: testCPUDemandSub(t, ps, fragmenter, tooBig), + subDomainConfig: testDomainConfigSub(t, ps), + cpuTopologyPinningSupported: true, + } + startThreadGranularApp(t, ctx, fragmenter) + + config := pinnedConfigForTest(tooBig.DisplayName, tooBig.CPUPlacement) + config.VmConfig.VCpus = tooBig.VCpus + var status types.DomainStatus + status.UUIDandVersion = config.UUIDandVersion + + err := assignCPUs(ctx, &config, &status) + var perr *placementError + if !errors.As(err, &perr) || perr.Code != types.ErrorCodeCPUPlacementInsufficient { + t.Fatalf("a request the plan cannot place either must stay unsatisfiable, "+ + "want %q, got %v", types.ErrorCodeCPUPlacementInsufficient, err) + } + if !strings.Contains(perr.Msg, "need 5 free cores") { + t.Errorf("the shortage explanation must be kept, got %q", perr.Msg) + } + // The condition a controller waits on has to carry the same counts as the + // message, taken from the allocator rather than recomputed, so the two can + // never disagree about how short the node is. + condition := placementErrorDescription(err).ErrorRetryCondition + for _, want := range []string{"5 whole physical cores", "0 free now"} { + if !strings.Contains(condition, want) { + t.Errorf("the retry condition must state the shortage (%q), got %q", + want, condition) + } + } + if status.PlacementQuality != types.CPUPlacementQualityUnspecified { + t.Errorf("a workload that never started has no placement quality, got %v", + status.PlacementQuality) + } +} + +// The classification table on its own, including the cases the end-to-end tests +// above cannot reach. +func TestLiveAllocationError_Classification(t *testing.T) { + id := uuid.NewV5(uuid.NamespaceOID, "classify") + fits := cpuallocator.Result{ + Status: cpuallocator.Success, + Assignment: &cpuallocator.Assignment{ + OrderedHostCPUs: []cputopology.LCPU{0, 4}, + }, + } + planned := map[uuid.UUID]cpuallocator.Result{id: fits} + unplanned := map[uuid.UUID]cpuallocator.Result{ + id: {Status: cpuallocator.Insufficient, Message: "need 5 free cores, have 4"}, + } + + for _, tt := range []struct { + name string + live cpuallocator.Status + plan map[uuid.UUID]cpuallocator.Result + want string + }{ + {"shortage the plan can place", cpuallocator.Insufficient, planned, + types.ErrorCodeCPUPlacementNeedsRepack}, + {"shortage the plan cannot place", cpuallocator.Insufficient, unplanned, + types.ErrorCodeCPUPlacementInsufficient}, + // An absent entry is a zero Result, whose Status happens to be Success. + {"workload not in the plan at all", cpuallocator.Insufficient, nil, + types.ErrorCodeCPUPlacementInsufficient}, + {"NUMA constraint the plan can meet", cpuallocator.NeedsRebalance, planned, + types.ErrorCodeCPUPlacementNeedsRepack}, + {"NUMA constraint, unplanned", cpuallocator.NeedsRebalance, nil, + types.ErrorCodeCPUPlacementNeedsRepack}, + // A caller bug must not be reported as something a repack would fix. + {"invalid request", cpuallocator.InvalidRequest, planned, + types.ErrorCodeCPUPolicyInvalid}, + } { + t.Run(tt.name, func(t *testing.T) { + err := liveAllocationError("app", id, + cpuallocator.Result{Status: tt.live, Message: "no room"}, tt.plan, nil) + if err.Code != tt.want { + t.Errorf("code = %q, want %q (%s)", err.Code, tt.want, err.Msg) + } + // Whatever the classification, the report has to say what would + // change it: the device never re-attempts placement, so an empty + // condition leaves a controller waiting for a recovery that will + // not come. + condition := placementErrorDescription(err).ErrorRetryCondition + if condition == "" { + t.Errorf("%s must carry a retry condition", err.Code) + } + if !strings.Contains(condition, "does not re-attempt") { + t.Errorf("the condition must not imply the device retries: %q", condition) + } + }) + } +} + +// Every code this path can publish must come with a retry condition, and one +// that names the action for that specific code. A structured "insufficient" with +// an empty retry_condition is what the controller cannot act on. +func TestPlacementErrorDescription_RetryConditionPerCode(t *testing.T) { + for _, tt := range []struct { + code string + // want is a phrase specific to this code's remedy, so a table that + // collapsed every code onto one generic sentence would fail here. + want string + }{ + {types.ErrorCodeCPUPlacementInsufficient, "Stop another pinned workload"}, + {types.ErrorCodeCPUPlacementNeedsRepack, "Restart the pinned workloads together"}, + {types.ErrorCodeCPUPolicyOddVCPU, "even number"}, + {types.ErrorCodeCPUIsolationTierUnavailable, "isolation_tier"}, + {types.ErrorCodeCPUTopologyUnsupported, "full_pcpus_only"}, + {types.ErrorCodeCPUPolicyInvalid, "CPU placement policy"}, + } { + t.Run(tt.code, func(t *testing.T) { + err := placementErrorf(tt.code, "something specific went wrong") + description := placementErrorDescription(err) + if description.ErrorCode != tt.code { + t.Errorf("code = %q, want %q", description.ErrorCode, tt.code) + } + if description.ErrorRetryCondition == "" { + t.Fatalf("%s published with an empty retry condition", tt.code) + } + if !strings.Contains(description.ErrorRetryCondition, tt.want) { + t.Errorf("condition for %s must mention %q, got %q", + tt.code, tt.want, description.ErrorRetryCondition) + } + // The message and the condition are shown in different places; a + // condition that just repeats the message says nothing new. + if strings.Contains(description.ErrorRetryCondition, err.Msg) { + t.Errorf("condition must not repeat the message: %q", + description.ErrorRetryCondition) + } + // Nothing on the device brings the workload back by itself, so the + // condition must not let a reader assume it will. + if !strings.Contains(description.ErrorRetryCondition, "does not re-attempt") { + t.Errorf("condition for %s must say placement is not re-attempted, got %q", + tt.code, description.ErrorRetryCondition) + } + }) + } +} + +// The retry condition is only useful if it survives the trip to the wire, where +// it is a field of its own next to the code. +func TestPlacementErrorDescription_ReachesErrorInfo(t *testing.T) { + err := placementErrorf(types.ErrorCodeCPUPlacementInsufficient, + "topology pinning for app: need 5 free cores, have 4") + + var et types.ErrorAndTime + et.SetErrorDescription(placementErrorDescription(err)) + errInfo := et.ToProto() + if errInfo == nil { + t.Fatal("a placement failure must produce an ErrorInfo") + } + if errInfo.ErrorCode != types.ErrorCodeCPUPlacementInsufficient { + t.Errorf("error_code = %q, want %q", errInfo.ErrorCode, + types.ErrorCodeCPUPlacementInsufficient) + } + if errInfo.RetryCondition == "" { + t.Error("retry_condition reached the controller empty") + } + if errInfo.RetryCondition != et.ErrorRetryCondition { + t.Errorf("retry_condition = %q, want %q", errInfo.RetryCondition, + et.ErrorRetryCondition) + } +} + +// A repack is only actionable if the report says which workloads have to be +// restarted, and a shortage only if it says how many cores short the node is. +func TestLiveAllocationError_RetryConditionsAreSpecific(t *testing.T) { + id := uuid.NewV5(uuid.NamespaceOID, "specific") + fits := map[uuid.UUID]cpuallocator.Result{id: { + Status: cpuallocator.Success, + Assignment: &cpuallocator.Assignment{ + OrderedHostCPUs: []cputopology.LCPU{0, 4}, + }, + }} + + repack := liveAllocationError("app", id, + cpuallocator.Result{Status: cpuallocator.Insufficient, Message: "no whole core free"}, + fits, []string{"noisy-a", "noisy-b"}) + condition := placementErrorDescription(repack).ErrorRetryCondition + for _, want := range []string{"noisy-a", "noisy-b", "repack"} { + if !strings.Contains(condition, want) { + t.Errorf("repack condition must mention %q, got %q", want, condition) + } + } + + shortage := liveAllocationError("app", id, cpuallocator.Result{ + Status: cpuallocator.Insufficient, + Message: "need 5 free cores, have 4", + CoresNeeded: 5, + CoresFree: 4, + }, nil, nil) + condition = placementErrorDescription(shortage).ErrorRetryCondition + for _, want := range []string{"5 whole physical cores", "4 free now"} { + if !strings.Contains(condition, want) { + t.Errorf("shortage condition must state the counts (%q), got %q", + want, condition) + } + } + + // The same shortage under a single-NUMA-node constraint clears under a + // different condition, and must say so rather than reuse the flat wording. + numa := liveAllocationError("app", id, cpuallocator.Result{ + Status: cpuallocator.NeedsRebalance, + Message: "need 2 cores in one NUMA node; none has enough (total free 4)", + CoresNeeded: 2, + CoresFree: 4, + }, nil, nil) + condition = placementErrorDescription(numa).ErrorRetryCondition + if !strings.Contains(condition, "single NUMA node") { + t.Errorf("a NUMA shortage must say the cores have to be on one node, got %q", + condition) + } +} + +// A CPU-placement failure is not re-attempted by the device. That is EVE's +// established behaviour for a workload whose assigned resource is unavailable -- +// an app that cannot get its PCI device is not started later because the device +// came back -- and CPUs follow it. The test pins the semantics so a future change +// towards auto-retry is a deliberate one, and states what the operator gets +// instead: a retry condition that says what to do. +func TestAssignCPUs_PlacementFailureStaysFailedUntilRestarted(t *testing.T) { + isolatePinningOverride(t) + cpuPlanFile = filepath.Join(t.TempDir(), "cpuplan.json") + + fragmenter := threadGranularDemand("thread-granular", 4) + wholeCore := wholeCoreDemand("wholecore", 2) + + ps := testPubSub(t) + statusPub := testPublication(t, ps, types.DomainStatus{}) + config := pinnedConfigForTest(wholeCore.DisplayName, wholeCore.CPUPlacement) + ctx := &domainContext{ + placer: testPlacer(t), + pubDomainStatus: statusPub, + subCPUDemandSet: testCPUDemandSub(t, ps, fragmenter, wholeCore), + subDomainConfig: testDomainConfigSub(t, ps, config), + cpuPinningSupported: true, + cpuTopologyPinningSupported: true, + } + startThreadGranularApp(t, ctx, fragmenter) + + var status types.DomainStatus + status.UUIDandVersion = config.UUIDandVersion + status.DomainName = config.DisplayName + + // The real activation path, which returns as soon as placement is refused -- + // before any hypervisor work -- and is where the decision not to retry lives. + config.Activate = true + doActivate(ctx, config, &status) + if !status.HasError() { + t.Fatalf("no whole core is free, activation must fail; got cpus=%v", + status.VmConfig.CPUs) + } + if status.ErrorCode != types.ErrorCodeCPUPlacementNeedsRepack { + t.Errorf("error code = %q, want %q", status.ErrorCode, + types.ErrorCodeCPUPlacementNeedsRepack) + } + + // Neither retry flag is set, which is what keeps maybeRetryBoot and + // maybeRetryConfig from picking the workload up on the 30s tick. + if status.BootFailed || status.ConfigFailed { + t.Errorf("a placement failure must not schedule an automatic retry, "+ + "BootFailed=%t ConfigFailed=%t", status.BootFailed, status.ConfigFailed) + } + + // The one automatic wakeup domainmgr has for a change in the dedicated CPU + // set must not place this workload either. It reaches only + // redistributeNonPinnedCPUs, which leaves anything pinned alone. + applied := cpuAllocationGen.Load() + triggerCPUNotification() + redistributeNonPinnedCPUs(ctx, config.Key(), applied) + if got := publishedCPUs(t, statusPub, status.Key()); len(got) != 0 { + t.Errorf("a CPU-set change must not silently place a failed workload, got %v", got) + } + item, err := statusPub.Get(status.Key()) + if err != nil { + t.Fatalf("no DomainStatus published: %v", err) + } + published := item.(types.DomainStatus) + if !published.HasError() { + t.Error("the failure must stand until the workload is acted on") + } + + // What the operator gets instead of a retry: the condition to act on. + if status.ErrorRetryCondition == "" { + t.Fatal("a failure that never self-heals must say what would fix it") + } + if !strings.Contains(status.ErrorRetryCondition, "does not re-attempt") { + t.Errorf("the report must not let a controller wait for a retry: %q", + status.ErrorRetryCondition) + } + + // Deactivation is what clears it. Until then handleModify refuses to + // activate a workload that HasError(), which is the whole of the "explicit + // restart" contract. + status.ClearError() + if status.HasError() { + t.Error("deactivation must clear the placement failure") + } +} + +func TestResolvePlacement_ThreadsPerCoreSelectsMode(t *testing.T) { + tests := []struct { + name string + policy types.CPUPlacementPolicy + wantMode cpuallocator.PinMode + }{ + { + name: "threads_per_core unset defaults to whole-core-smt", + policy: types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + }, + wantMode: cpuallocator.ModeWholeCoreSMT, + }, + { + name: "threads_per_core=2 is whole-core-smt", + policy: types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, ThreadsPerCore: 2, + }, + wantMode: cpuallocator.ModeWholeCoreSMT, + }, + { + name: "threads_per_core=1 parks the sibling", + policy: types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, ThreadsPerCore: 1, + }, + wantMode: cpuallocator.ModeOnePerCore, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := resolvePlacement(tt.policy) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !got.TopologyAware { + t.Fatal("expected topology-aware placement") + } + if got.Mode != tt.wantMode { + t.Errorf("mode = %v, want %v", got.Mode, tt.wantMode) + } + }) + } +} + +func TestResolvePlacement_NUMAMapping(t *testing.T) { + tests := []struct { + in types.CPUNUMAPolicy + want cpuallocator.NUMAPolicy + }{ + {types.CPUNUMAPolicyUnspecified, cpuallocator.NUMABestEffort}, + {types.CPUNUMAPolicyBestEffort, cpuallocator.NUMABestEffort}, + {types.CPUNUMAPolicyNone, cpuallocator.NUMAAllowCross}, + {types.CPUNUMAPolicyRestricted, cpuallocator.NUMALocal}, + {types.CPUNUMAPolicySingleNode, cpuallocator.NUMALocal}, + } + for _, tt := range tests { + t.Run(tt.in.String(), func(t *testing.T) { + got, err := resolvePlacement(types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, NUMAPolicy: tt.in, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.NUMA != tt.want { + t.Errorf("numa = %v, want %v", got.NUMA, tt.want) + } + }) + } +} + +// Dedicated without full-pcpus-only is thread-granular: it still pins, but via +// the legacy shared-pool allocator, not topology-aware placement. +func TestResolvePlacement_DedicatedWithoutFullPCPUsIsNotTopologyAware(t *testing.T) { + got, err := resolvePlacement(types.CPUPlacementPolicy{Policy: types.CPUPolicyDedicated}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.TopologyAware { + t.Error("dedicated without full-pcpus-only must not be topology-aware") + } +} + +func TestResolvePlacement_IOPlacement(t *testing.T) { + base := types.CPUPlacementPolicy{Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true} + + for _, tt := range []struct { + name string + io types.CPUIOPlacement + want bool + }{ + {"unspecified defaults to dedicated", types.CPUIOPlacementUnspecified, false}, + {"dedicated", types.CPUIOPlacementDedicated, false}, + {"housekeeping", types.CPUIOPlacementHousekeeping, true}, + } { + t.Run(tt.name, func(t *testing.T) { + p := base + p.IOPlacement = tt.io + got, err := resolvePlacement(p) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.IOHousekeeping != tt.want { + t.Errorf("IOHousekeeping = %v, want %v", got.IOHousekeeping, tt.want) + } + }) + } +} + +// The hard tier needs a kernel command-line change this device cannot make at +// runtime. It must fail closed with a structured code rather than quietly +// running with only soft isolation. +func TestResolvePlacement_HardIsolationFailsClosed(t *testing.T) { + _, err := resolvePlacement(types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, + FullPCPUsOnly: true, + IsolationTier: types.CPUIsolationTierHard, + }) + if err == nil { + t.Fatal("hard isolation must be rejected") + } + var perr *placementError + if !errors.As(err, &perr) { + t.Fatalf("expected a placementError carrying an error code, got %T", err) + } + if perr.Code != types.ErrorCodeCPUIsolationTierUnavailable { + t.Errorf("code = %q, want %q", perr.Code, types.ErrorCodeCPUIsolationTierUnavailable) + } +} + +func TestResolvePlacement_SoftAndNoneTiersAccepted(t *testing.T) { + for _, tier := range []types.CPUIsolationTier{ + types.CPUIsolationTierUnspecified, + types.CPUIsolationTierNone, + types.CPUIsolationTierSoft, + } { + t.Run(tier.String(), func(t *testing.T) { + if _, err := resolvePlacement(types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, IsolationTier: tier, + }); err != nil { + t.Errorf("tier %v should be accepted: %v", tier, err) + } + }) + } +} + +// "protect" asks the node to defer disruptive actions, which nothing on the +// device does. Accepting it silently would hand the controller a guarantee that +// does not exist. +func TestResolvePlacement_ProtectDisruptionFailsClosed(t *testing.T) { + _, err := resolvePlacement(types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, + FullPCPUsOnly: true, + DisruptionPolicy: types.CPUDisruptionPolicyProtect, + }) + if err == nil { + t.Fatal("an unimplemented disruption policy must be rejected") + } + var perr *placementError + if !errors.As(err, &perr) || perr.Code != types.ErrorCodeCPUPolicyInvalid { + t.Fatalf("expected %q, got %v", types.ErrorCodeCPUPolicyInvalid, err) + } +} + +func TestResolvePlacement_AllowAndUnspecifiedDisruptionAccepted(t *testing.T) { + for _, policy := range []types.CPUDisruptionPolicy{ + types.CPUDisruptionPolicyUnspecified, + types.CPUDisruptionPolicyAllow, + } { + t.Run(policy.String(), func(t *testing.T) { + if _, err := resolvePlacement(types.CPUPlacementPolicy{ + Policy: types.CPUPolicyDedicated, FullPCPUsOnly: true, + DisruptionPolicy: policy, + }); err != nil { + t.Errorf("disruption policy %v should be accepted: %v", policy, err) + } + }) + } +} + +// The /persist override is the only way to ask for whole-core placement on a +// device whose controller knows nothing about the policy. Since the override +// cannot set the app config's pin flag, asking for it there must imply pinning +// -- otherwise the file is silently inert, which is what it was. +func TestEffectiveCPUsPinned_PersistOverrideImpliesPinning(t *testing.T) { + dir := t.TempDir() + pinConfigDir = dir + pinConfigFile = filepath.Join(dir, "config.json") + + id := uuid.NewV5(uuid.NamespaceOID, "override") + var config types.DomainConfig + config.UUIDandVersion.UUID = id + config.VmConfig.VCpus = 2 + + if effectiveCPUsPinned(&config) { + t.Fatal("no controller policy and no override must not pin") + } + writePinningEntryForTest(t, id, &PinningEntry{ + CPUPolicy: "static", PolicyOptions: fullPCPUs(), + }) + if !effectiveCPUsPinned(&config) { + t.Error("a whole-core override must imply pinning") + } + + // The controller stays authoritative whenever it said anything, so a stale + // local file cannot contradict it. + config.VmConfig.CPUPlacement.Policy = types.CPUPolicyShared + if effectiveCPUsPinned(&config) { + t.Error("an explicit shared policy from the controller must win over the override") + } + config.VmConfig.CPUPlacement.Policy = types.CPUPolicyUnspecified + + // An override that does not ask for whole cores keeps the legacy behavior: + // pinning stays off unless the controller asked for it. + writePinningEntryForTest(t, id, &PinningEntry{CPUPolicy: "static"}) + if effectiveCPUsPinned(&config) { + t.Error("static without full-pcpus-only must not imply pinning") + } + config.VmConfig.CPUsPinned = true + if !effectiveCPUsPinned(&config) { + t.Error("the controller's pin flag must always pin") + } +} + +// End to end through assignCPUs: an override-only workload must take the +// topology path. It is refused here only because this context reports no +// per-vCPU pinning support -- before, it silently fell through to the +// "not pinned" branch and got the whole shared CPU set. +func TestAssignCPUs_PersistOverrideTakesTopologyPath(t *testing.T) { + dir := t.TempDir() + pinConfigDir = dir + pinConfigFile = filepath.Join(dir, "config.json") + + id := uuid.NewV5(uuid.NamespaceOID, "override-assign") + writePinningEntryForTest(t, id, &PinningEntry{ + CPUPolicy: "static", PolicyOptions: fullPCPUs(), + }) + var config types.DomainConfig + config.UUIDandVersion.UUID = id + config.DisplayName = "override-assign" + config.VmConfig.VCpus = 2 + + ctx := &domainContext{placer: testPlacer(t), cpuTopologyPinningSupported: false} + var status types.DomainStatus + err := assignCPUs(ctx, &config, &status) + + var perr *placementError + if !errors.As(err, &perr) || perr.Code != types.ErrorCodeCPUTopologyUnsupported { + t.Fatalf("override must be resolved as a whole-core request, got err=%v cpus=%v", + err, status.VmConfig.CPUs) + } + if !status.VmConfig.CPUsPinned { + t.Error("status must record that the workload is pinned") + } +} + +// whole-core-smt draws two vCPUs from every physical core, so an odd count can +// never be satisfied. The controller validates this too, but the device keeps a +// backstop so a hand-written config cannot half-start a VM. +func TestValidateVCPUCount_OddRejectedForWholeCoreSMT(t *testing.T) { + smt := resolvedPlacement{TopologyAware: true, Mode: cpuallocator.ModeWholeCoreSMT} + if err := validateVCPUCount(smt, 3); err == nil { + t.Fatal("odd vCPU count must be rejected for whole-core-smt") + } else { + var perr *placementError + if !errors.As(err, &perr) || perr.Code != types.ErrorCodeCPUPolicyOddVCPU { + t.Errorf("expected %q, got %v", types.ErrorCodeCPUPolicyOddVCPU, err) + } + } + if err := validateVCPUCount(smt, 4); err != nil { + t.Errorf("even count must be accepted: %v", err) + } + // one-per-core has no parity constraint. + ope := resolvedPlacement{TopologyAware: true, Mode: cpuallocator.ModeOnePerCore} + if err := validateVCPUCount(ope, 3); err != nil { + t.Errorf("one-per-core must accept an odd count: %v", err) + } +} From 7c5240ea8ca5c1ab6a7cbea55d7444735018771e Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:04:59 +0000 Subject: [PATCH 09/15] evetest: give tests a CPU topology to test against and the means to observe it A test cannot assert anything about CPU placement on a node whose CPU topology it does not control: with a flat single-thread-per-core VM every placement policy looks alike, and whole-core, one-per-core and parked-sibling behaviour are indistinguishable. The device requirements gain a threads-per-core knob, so a test can ask for a node with real SMT siblings, and the QEMU and libvirt providers derive the -smp topology from the requested CPU count and that knob through one shared helper -- the two providers disagreeing about what a requirement means would make results depend on which one ran. On the observation side, tests get the node facts CPU placement work needs: the host's socket/core/sibling topology, which CPUs are online, which the kernel isolated, and the kernel command line, so an expectation can be stated in terms of what the node actually is rather than hard-coded numbers. Per-workload facts come from QMP over the existing SSH transport. The guest-vCPU-to-host-thread mapping is only available there -- QEMU does not name its vCPU threads, and a domain's thread group contains helper threads that cannot be told apart from vCPU threads by name. A QMP call is also a point-in-time question with a definite answer, which is what an assertion wants, where waiting for a log line is a race dressed up as a check. The call is bounded by closing the connection from a timer, since deadlines are not supported on SSH channels. The application config gains the CPU placement policy fields and a start delay. The delay exists to test the property that matters most here: that placement does not depend on the order workloads happen to start in. Signed-off-by: Mikhail Malyshev --- evetest/broker/broker.go | 1 + evetest/broker/provider/common.go | 29 ++ evetest/broker/provider/cputopology_test.go | 52 ++++ evetest/broker/provider/libvirt.go | 13 +- evetest/broker/provider/provider.go | 7 + evetest/broker/provider/qemu.go | 2 +- evetest/devconfig.go | 61 +++- evetest/grpcapi/go/common.pb.go | 39 ++- evetest/grpcapi/proto/common.proto | 7 + evetest/hostcpu.go | 318 ++++++++++++++++++++ evetest/qmp.go | 142 +++++++++ evetest/requirements.go | 11 + evetest/setup.go | 21 +- 13 files changed, 672 insertions(+), 31 deletions(-) create mode 100644 evetest/broker/provider/cputopology_test.go create mode 100644 evetest/hostcpu.go create mode 100644 evetest/qmp.go diff --git a/evetest/broker/broker.go b/evetest/broker/broker.go index 7c7f923655b..41994d54397 100644 --- a/evetest/broker/broker.go +++ b/evetest/broker/broker.go @@ -1120,6 +1120,7 @@ func (b *broker) SetupDevices( // Build EVE device specification for the provider. dev.CPUs = uint(eveDevice.GetCpus()) + dev.ThreadsPerCore = uint(eveDevice.GetThreadsPerCore()) dev.MemoryBytes = eveDevice.GetMemoryBytes() dev.WithTPM = eveDevice.GetWithTpm() dev.SerialNumber = eveDevice.GetSerialNumber() diff --git a/evetest/broker/provider/common.go b/evetest/broker/provider/common.go index 6450c2c3f7e..1f7becb33bc 100644 --- a/evetest/broker/provider/common.go +++ b/evetest/broker/provider/common.go @@ -18,6 +18,35 @@ import ( "github.com/vishvananda/netlink" ) +// CPUTopology converts a device spec's CPU count and requested threads-per-core +// into a concrete sockets/cores/threads layout for the hypervisor. +// +// A guest can only reason about SMT if it is told which logical CPUs share a +// physical core, and neither QEMU nor libvirt infers that: left alone both +// present every CPU as its own single-thread core. Keeping the arithmetic here +// means the qemu and libvirt providers cannot disagree about it. +// +// The default (threads-per-core of zero or one) reproduces exactly what the +// providers did before this existed: one socket, one core per CPU, one thread +// per core. A requested value that does not divide the CPU count is ignored +// rather than silently rounded, since a partial core is not a thing a guest can +// be shown. +func CPUTopology(cpus, threadsPerCore uint) (sockets, cores, threads uint) { + if threadsPerCore < 2 || cpus%threadsPerCore != 0 { + return 1, cpus, 1 + } + return 1, cpus / threadsPerCore, threadsPerCore +} + +// smpArg renders the QEMU -smp value for a device's CPU layout. QEMU expands a +// bare "-smp N" with threads=1, so the topology has to be stated explicitly for +// the guest to see sibling threads at all. +func smpArg(cpus, threadsPerCore uint) string { + sockets, cores, threads := CPUTopology(cpus, threadsPerCore) + return fmt.Sprintf("%d,sockets=%d,cores=%d,threads=%d", + cpus, sockets, cores, threads) +} + // namePrefix is prepended to all domain (VM) names created by a device // provider. This ensures that test/dev resources are clearly separated from // user-managed objects. When listing devices, the prefix is stripped so that diff --git a/evetest/broker/provider/cputopology_test.go b/evetest/broker/provider/cputopology_test.go new file mode 100644 index 00000000000..5809d41a26a --- /dev/null +++ b/evetest/broker/provider/cputopology_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package provider + +import "testing" + +func TestCPUTopology(t *testing.T) { + tests := []struct { + name string + cpus, threadsPerCore uint + wantSockets, wantCores, wantThr uint + }{ + // The default must reproduce exactly what the providers did before + // threads-per-core existed: every CPU its own single-thread core. + {"unset is one thread per core", 8, 0, 1, 8, 1}, + {"explicit 1 is one thread per core", 8, 1, 1, 8, 1}, + + {"SMT halves the core count", 8, 2, 1, 4, 2}, + {"SMT with the minimum core count", 2, 2, 1, 1, 2}, + + // A partial core cannot be presented to a guest, so an indivisible + // request degrades to the default rather than being rounded. + {"odd CPU count cannot be SMT", 7, 2, 1, 7, 1}, + {"four threads per core", 8, 4, 1, 2, 4}, + {"threads exceeding CPUs degrade", 2, 4, 1, 2, 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sockets, cores, threads := CPUTopology(tt.cpus, tt.threadsPerCore) + if sockets != tt.wantSockets || cores != tt.wantCores || threads != tt.wantThr { + t.Errorf("CPUTopology(%d, %d) = %d/%d/%d, want %d/%d/%d", + tt.cpus, tt.threadsPerCore, sockets, cores, threads, + tt.wantSockets, tt.wantCores, tt.wantThr) + } + // However the CPUs are arranged, the guest must still end up with + // the number of logical CPUs it asked for. + if got := sockets * cores * threads; got != tt.cpus { + t.Errorf("topology describes %d logical CPUs, want %d", got, tt.cpus) + } + }) + } +} + +func TestSMPArg(t *testing.T) { + if got, want := smpArg(8, 0), "8,sockets=1,cores=8,threads=1"; got != want { + t.Errorf("smpArg(8, 0) = %q, want %q", got, want) + } + if got, want := smpArg(8, 2), "8,sockets=1,cores=4,threads=2"; got != want { + t.Errorf("smpArg(8, 2) = %q, want %q", got, want) + } +} diff --git a/evetest/broker/provider/libvirt.go b/evetest/broker/provider/libvirt.go index 3c4b43d25bd..07a6b2224bf 100644 --- a/evetest/broker/provider/libvirt.go +++ b/evetest/broker/provider/libvirt.go @@ -410,11 +410,14 @@ func (p *LibvirtProvider) SetupDevice( }, CPU: &libvirtxml.DomainCPU{ Mode: "host-passthrough", // expose host CPU features - Topology: &libvirtxml.DomainCPUTopology{ - Sockets: 1, - Cores: int(spec.CPUs), - Threads: 1, - }, + Topology: func() *libvirtxml.DomainCPUTopology { + sockets, cores, threads := CPUTopology(spec.CPUs, spec.ThreadsPerCore) + return &libvirtxml.DomainCPUTopology{ + Sockets: int(sockets), + Cores: int(cores), + Threads: int(threads), + } + }(), }, Features: features, Devices: devices, diff --git a/evetest/broker/provider/provider.go b/evetest/broker/provider/provider.go index d7d57064ae1..89d9e0a31d0 100644 --- a/evetest/broker/provider/provider.go +++ b/evetest/broker/provider/provider.go @@ -173,6 +173,13 @@ type DeviceSpec struct { // CPUs is the number of CPU cores to allocate. CPUs uint + // ThreadsPerCore is how many SMT hardware threads each physical core + // exposes to the guest. Zero or one -- the default -- gives every CPU its + // own single-thread core. Higher values arrange the same number of CPUs as + // CPUs/ThreadsPerCore cores with sibling threads, which a guest needs + // before it can make SMT-aware placement decisions. + ThreadsPerCore uint + // MemoryBytes is the amount of RAM in bytes. MemoryBytes uint64 diff --git a/evetest/broker/provider/qemu.go b/evetest/broker/provider/qemu.go index e8ef2a72ade..0a80f2e5352 100644 --- a/evetest/broker/provider/qemu.go +++ b/evetest/broker/provider/qemu.go @@ -1490,7 +1490,7 @@ func (dev *qemuDevice) buildArgs() []string { "-enable-kvm", "-machine", "q35", "-cpu", "host", - "-smp", fmt.Sprintf("%d", dev.spec.CPUs), + "-smp", smpArg(dev.spec.CPUs, dev.spec.ThreadsPerCore), "-m", fmt.Sprintf("%d", dev.spec.MemoryBytes>>20), "-nographic", } diff --git a/evetest/devconfig.go b/evetest/devconfig.go index 4793dab64f2..bf20c3ed851 100644 --- a/evetest/devconfig.go +++ b/evetest/devconfig.go @@ -903,9 +903,46 @@ type ApplicationInstanceConfig struct { // rewire) the app's VolumeRefList to point at it; they never create or // remove the volume itself. Mounts []MountConfig + // CPUPlacement is the application's CPU placement intent. Its zero value + // sends no policy at all, so an application that does not set it keeps + // today's best-effort placement. + CPUPlacement CPUPlacementConfig + // StartDelayInSeconds holds this application back for that long while the + // others start normally. + // + // The delay is counted from the moment EVE first obtains configuration + // after booting, not from when this field is set, so setting it on an + // already-running application changes nothing until the next boot. That is + // what makes it usable as a test instrument: it is the only way to give a + // device a deterministic, staggered application start order without + // changing which applications are configured. + StartDelayInSeconds uint32 // Many more parameters can be configured; they will be added later as needed. } +// CPUPlacementConfig expresses what an application needs from CPU placement: +// whether it gets host CPUs of its own, whether those must be whole physical +// cores, how its vCPUs map onto SMT siblings, and where the hypervisor's +// emulator/IO threads run. +// +// The device picks the concrete host CPUs -- the controller only states intent +// -- so there is deliberately no way here to name specific CPU ids. Assertions +// about which CPUs a workload actually got are made against what the device +// reports back, or read off the device directly. +type CPUPlacementConfig struct { + // Policy is the on-switch: CPU_POLICY_DEDICATED activates the rest. + Policy eveconfig.CpuPolicy + FullPCPUsOnly bool + // ThreadsPerCore selects how many SMT siblings of each dedicated core + // become vCPUs: 2 (whole-core-SMT, the default when unset) or 1 + // (one-per-core, sibling parked idle). + ThreadsPerCore uint32 + NUMAPolicy eveconfig.NumaPolicy + IOPlacement eveconfig.IoPlacement + IsolationTier eveconfig.IsolationTier + DisruptionPolicy eveconfig.DisruptionPolicy +} + // MountConfig attaches an existing volume to an application, in addition to // its root disk. The volume must already exist in the device configuration // (created via EdgeDeviceConfig.AddVolume or AddBlankVolume) -- @@ -942,6 +979,13 @@ func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string, VncPasswd: config.VNCPassword, DisableLogs: config.DisableLogs, EnforceNetworkInterfaceOrder: config.EnforceNetIntfOrder, + CpuPolicy: config.CPUPlacement.Policy, + FullPcpusOnly: config.CPUPlacement.FullPCPUsOnly, + ThreadsPerCore: config.CPUPlacement.ThreadsPerCore, + NumaPolicy: config.CPUPlacement.NUMAPolicy, + IoPlacement: config.CPUPlacement.IOPlacement, + IsolationTier: config.CPUPlacement.IsolationTier, + DisruptionPolicy: config.CPUPlacement.DisruptionPolicy, } if config.MemoryBytes != 0 { vmConfig.Memory = uint32(config.MemoryBytes / KiB) @@ -951,11 +995,12 @@ func (config ApplicationInstanceConfig) toProto(th *TestHarness, devName string, Uuid: appUUID.String(), Version: "1", }, - Displayname: config.DisplayName, - Fixedresources: vmConfig, - Activate: config.Activate, - ProfileList: config.ProfileList, - RemoteConsole: config.RemoteConsole, + Displayname: config.DisplayName, + Fixedresources: vmConfig, + Activate: config.Activate, + ProfileList: config.ProfileList, + RemoteConsole: config.RemoteConsole, + StartDelayInSeconds: config.StartDelayInSeconds, } if volumeUUID != NilUUID { appInstConfig.VolumeRefList = append(appInstConfig.VolumeRefList, @@ -2391,7 +2436,7 @@ func (dc *EdgeDeviceConfig) buildMountRefs( func (dc *EdgeDeviceConfig) UpdateApplication( appUUID uuid.UUID, newConfig ApplicationInstanceConfig) { // For now, we will only allow to change Activation flag, profile list, - // adapters, and mounts. + // adapters, mounts, and the start delay. for i, app := range dc.Apps { if app.Uuidandversion.Uuid == appUUID.String() { newProtoConfig := newConfig.toProto(dc.th, dc.DeviceName, appUUID, NilUUID) @@ -2442,6 +2487,10 @@ func (dc *EdgeDeviceConfig) UpdateApplication( app.Restart.Counter++ } dc.Apps[i].Activate = newProtoConfig.Activate + // The start delay is not a fixed resource: it only decides when the + // application is released on the next boot, so it can be changed on + // a deployed application without a purge or a restart. + dc.Apps[i].StartDelayInSeconds = newProtoConfig.StartDelayInSeconds dc.Apps[i].ProfileList = newProtoConfig.ProfileList dc.Apps[i].Adapters = newProtoConfig.Adapters dc.Apps[i].Interfaces = newProtoConfig.Interfaces diff --git a/evetest/grpcapi/go/common.pb.go b/evetest/grpcapi/go/common.pb.go index 4ac578a8450..3f774cf7962 100644 --- a/evetest/grpcapi/go/common.pb.go +++ b/evetest/grpcapi/go/common.pb.go @@ -551,16 +551,23 @@ func (x *LogMessage) GetTimestamp() *timestamppb.Timestamp { // Describes a single EVE device instance managed by the test framework. type EVEDevice struct { - state protoimpl.MessageState `protogen:"open.v1"` - DeviceName string `protobuf:"bytes,1,opt,name=device_name,json=deviceName,proto3" json:"device_name,omitempty"` - Cpus uint32 `protobuf:"varint,2,opt,name=cpus,proto3" json:"cpus,omitempty"` - MemoryBytes uint64 `protobuf:"varint,3,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` - WithTpm bool `protobuf:"varint,4,opt,name=with_tpm,json=withTpm,proto3" json:"with_tpm,omitempty"` - SerialNumber string `protobuf:"bytes,5,opt,name=serial_number,json=serialNumber,proto3" json:"serial_number,omitempty"` - Interfaces []*EVEInterface `protobuf:"bytes,6,rep,name=interfaces,proto3" json:"interfaces,omitempty"` - Image *ImageRef `protobuf:"bytes,7,opt,name=image,proto3" json:"image,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + DeviceName string `protobuf:"bytes,1,opt,name=device_name,json=deviceName,proto3" json:"device_name,omitempty"` + Cpus uint32 `protobuf:"varint,2,opt,name=cpus,proto3" json:"cpus,omitempty"` + MemoryBytes uint64 `protobuf:"varint,3,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + WithTpm bool `protobuf:"varint,4,opt,name=with_tpm,json=withTpm,proto3" json:"with_tpm,omitempty"` + SerialNumber string `protobuf:"bytes,5,opt,name=serial_number,json=serialNumber,proto3" json:"serial_number,omitempty"` + Interfaces []*EVEInterface `protobuf:"bytes,6,rep,name=interfaces,proto3" json:"interfaces,omitempty"` + Image *ImageRef `protobuf:"bytes,7,opt,name=image,proto3" json:"image,omitempty"` + // How many SMT threads each physical core of the device exposes. + // Zero or one means no SMT: the device sees `cpus` single-thread cores, + // which is the default. Two makes the device see cpus/2 physical cores + // with two hardware threads each, which is what a test exercising + // SMT-aware CPU placement needs -- without it the guest has no sibling + // thread to place a second vCPU on. + ThreadsPerCore uint32 `protobuf:"varint,8,opt,name=threads_per_core,json=threadsPerCore,proto3" json:"threads_per_core,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EVEDevice) Reset() { @@ -642,6 +649,13 @@ func (x *EVEDevice) GetImage() *ImageRef { return nil } +func (x *EVEDevice) GetThreadsPerCore() uint32 { + if x != nil { + return x.ThreadsPerCore + } + return 0 +} + // Reports the current runtime status of an EVE device. type EVEDeviceStatus struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1010,7 +1024,7 @@ const file_common_proto_rawDesc = "" + "\amessage\x18\x01 \x01(\tR\amessage\x12;\n" + "\bseverity\x18\x02 \x01(\x0e2\x1f.org.lfedge.evetest.LogSeverityR\bseverity\x12\x16\n" + "\x06source\x18\x03 \x01(\tR\x06source\x128\n" + - "\ttimestamp\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\"\x99\x02\n" + + "\ttimestamp\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\"\xc3\x02\n" + "\tEVEDevice\x12\x1f\n" + "\vdevice_name\x18\x01 \x01(\tR\n" + "deviceName\x12\x12\n" + @@ -1021,7 +1035,8 @@ const file_common_proto_rawDesc = "" + "\n" + "interfaces\x18\x06 \x03(\v2 .org.lfedge.evetest.EVEInterfaceR\n" + "interfaces\x122\n" + - "\x05image\x18\a \x01(\v2\x1c.org.lfedge.evetest.ImageRefR\x05image\"\xc6\x01\n" + + "\x05image\x18\a \x01(\v2\x1c.org.lfedge.evetest.ImageRefR\x05image\x12(\n" + + "\x10threads_per_core\x18\b \x01(\rR\x0ethreadsPerCore\"\xc6\x01\n" + "\x0fEVEDeviceStatus\x121\n" + "\x04spec\x18\x01 \x01(\v2\x1d.org.lfedge.evetest.EVEDeviceR\x04spec\x128\n" + "\x05state\x18\x02 \x01(\x0e2\".org.lfedge.evetest.EVEDeviceStateR\x05state\x12F\n" + diff --git a/evetest/grpcapi/proto/common.proto b/evetest/grpcapi/proto/common.proto index 507973c87eb..58bf36851ff 100644 --- a/evetest/grpcapi/proto/common.proto +++ b/evetest/grpcapi/proto/common.proto @@ -93,6 +93,13 @@ message EVEDevice { string serial_number = 5; repeated EVEInterface interfaces = 6; ImageRef image = 7; + // How many SMT threads each physical core of the device exposes. + // Zero or one means no SMT: the device sees `cpus` single-thread cores, + // which is the default. Two makes the device see cpus/2 physical cores + // with two hardware threads each, which is what a test exercising + // SMT-aware CPU placement needs -- without it the guest has no sibling + // thread to place a second vCPU on. + uint32 threads_per_core = 8; } // Represents the current lifecycle state of an EVE device. diff --git a/evetest/hostcpu.go b/evetest/hostcpu.go new file mode 100644 index 00000000000..c7961f17c5b --- /dev/null +++ b/evetest/hostcpu.go @@ -0,0 +1,318 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package evetest + +import ( + "fmt" + "strconv" + "strings" + "time" + + uuid "github.com/satori/go.uuid" + + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// This file exposes the device's CPU reality: how its CPUs are laid out, which +// of them the kernel has been told to treat specially, and where a workload's +// threads have actually ended up. +// +// It exists so that tests about CPU placement and isolation assert against +// typed values rather than each growing its own shell script. Everything here +// reads standard Linux interfaces (/sys/devices/system/cpu, /proc) or the +// hypervisor's own monitor, never EVE-internal state, so these helpers stay +// valid across EVE versions and remain independent of the code under test -- +// which is what makes them usable as evidence. + +const sysfsCPURoot = "/sys/devices/system/cpu" + +// hostQueryTimeout bounds each of the small shell reads below. +const hostQueryTimeout = 30 * time.Second + +// HostCPU is one logical CPU of the device and its position in the CPU +// topology. The identifiers are opaque grouping keys: equal values mean "same +// domain", and nothing more should be read into them -- they are not guaranteed +// contiguous or zero-based. +type HostCPU struct { + // ID is the logical CPU number the kernel schedules on and that CPU + // affinities are expressed in. + ID uint32 + // Socket is the physical package. + Socket uint32 + // Core identifies the physical core within the socket. Logical CPUs that + // share a (Socket, Core) pair are SMT siblings of one physical core. + Core uint32 + // Siblings are all logical CPUs on this CPU's physical core, including + // itself. A single entry means SMT is off or unavailable for this core. + Siblings []uint32 +} + +// HostTopology is the device's CPU topology, indexed by logical CPU. +type HostTopology struct { + CPUs map[uint32]HostCPU +} + +// SameCore reports whether two logical CPUs are SMT siblings on one physical +// core. Unknown CPUs are reported as not sharing a core. +// +// This is the question a whole-core placement test actually needs answered: an +// allocation that hands a workload two CPUs is only correct for +// full_pcpus_only + threads_per_core=1 if those CPUs are on *different* cores, +// and only correct for threads_per_core=2 if the pairs are on the *same* core. +func (t HostTopology) SameCore(a, b uint32) bool { + ca, okA := t.CPUs[a] + cb, okB := t.CPUs[b] + if !okA || !okB { + return false + } + return ca.Socket == cb.Socket && ca.Core == cb.Core +} + +// SiblingsOf returns the logical CPUs sharing a physical core with the given +// CPU, including itself, or nil if the CPU is unknown. +func (t HostTopology) SiblingsOf(cpu uint32) []uint32 { + return t.CPUs[cpu].Siblings +} + +// IDs returns every known logical CPU id, ascending. +func (t HostTopology) IDs() []uint32 { + ids := make([]uint32, 0, len(t.CPUs)) + for id := range t.CPUs { + ids = append(ids, id) + } + sortCPUs(ids) + return ids +} + +// HostCPUTopology reads the device's CPU topology from sysfs. +func (d *EdgeDevice) HostCPUTopology() (HostTopology, error) { + script := fmt.Sprintf(`for dir in %s/cpu[0-9]*; do + id=${dir#%s/cpu} + echo "$id $(cat "$dir/topology/physical_package_id" 2>/dev/null) $(cat "$dir/topology/core_id" 2>/dev/null) $(cat "$dir/topology/thread_siblings_list" 2>/dev/null)" +done`, sysfsCPURoot, sysfsCPURoot) + + stdout, stderr, err := d.RunShellScript(script, hostQueryTimeout, 0) + if err != nil { + return HostTopology{}, fmt.Errorf("failed to read CPU topology: %w (stderr: %s)", + err, stderr) + } + + topo := HostTopology{CPUs: map[uint32]HostCPU{}} + for _, line := range strings.Split(strings.TrimSpace(stdout), "\n") { + fields := strings.Fields(line) + if len(fields) < 4 { + continue + } + id, err := strconv.ParseUint(fields[0], 10, 32) + if err != nil { + continue + } + // A hypervisor may report no package id at all; treat that as a single + // socket rather than discarding the CPU. + socket, err := strconv.ParseUint(fields[1], 10, 32) + if err != nil { + socket = 0 + } + core, err := strconv.ParseUint(fields[2], 10, 32) + if err != nil { + continue + } + topo.CPUs[uint32(id)] = HostCPU{ + ID: uint32(id), + Socket: uint32(socket), + Core: uint32(core), + Siblings: ParseCPUList(fields[3]), + } + } + if len(topo.CPUs) == 0 { + return topo, fmt.Errorf("no CPU topology found under %s", sysfsCPURoot) + } + return topo, nil +} + +// OnlineCPUs returns the logical CPUs the kernel currently has online. +func (d *EdgeDevice) OnlineCPUs() ([]uint32, error) { + return d.readCPUListFile(sysfsCPURoot + "/online") +} + +// IsolatedCPUs returns the logical CPUs the running kernel is isolating from the +// scheduler, i.e. the effective result of the isolcpus boot parameter. It is +// empty on a node with no kernel-level isolation. +// +// This reads what the kernel is actually doing rather than parsing the command +// line, so it reflects reality even when the parameter was malformed, capped, or +// supplied by some other means. +func (d *EdgeDevice) IsolatedCPUs() ([]uint32, error) { + return d.readCPUListFile(sysfsCPURoot + "/isolated") +} + +// NohzFullCPUs returns the logical CPUs running without the scheduler tick, i.e. +// the effective result of the nohz_full boot parameter. +func (d *EdgeDevice) NohzFullCPUs() ([]uint32, error) { + return d.readCPUListFile(sysfsCPURoot + "/nohz_full") +} + +// KernelCmdline returns the device's kernel command line. +func (d *EdgeDevice) KernelCmdline() (string, error) { + stdout, stderr, err := d.RunShellScript("cat /proc/cmdline", hostQueryTimeout, 0) + if err != nil { + return "", fmt.Errorf("failed to read kernel command line: %w (stderr: %s)", + err, stderr) + } + return strings.TrimSpace(stdout), nil +} + +// KernelCmdlineParam returns the value of a kernel command-line parameter and +// whether it is present. A parameter given without a value (a bare flag) is +// reported as present with an empty value. +func (d *EdgeDevice) KernelCmdlineParam(name string) (string, bool, error) { + cmdline, err := d.KernelCmdline() + if err != nil { + return "", false, err + } + for _, field := range strings.Fields(cmdline) { + key, value, hasValue := strings.Cut(field, "=") + if key != name { + continue + } + if !hasValue { + return "", true, nil + } + return value, true, nil + } + return "", false, nil +} + +// ThreadAffinity returns the logical CPUs a single host thread is allowed to run +// on, from /proc//status. An empty result means the thread is gone. +func (d *EdgeDevice) ThreadAffinity(tid int) ([]uint32, error) { + script := fmt.Sprintf( + `awk '/Cpus_allowed_list/{print $2}' /proc/%d/status 2>/dev/null`, tid) + stdout, stderr, err := d.RunShellScript(script, hostQueryTimeout, 0) + if err != nil { + return nil, fmt.Errorf("failed to read affinity of thread %d: %w (stderr: %s)", + tid, err, stderr) + } + return ParseCPUList(stdout), nil +} + +// AppDomainName returns the hypervisor domain name of a deployed application, +// which identifies it to the hypervisor and locates its monitor socket. +func (d *EdgeDevice) AppDomainName(appUUID uuid.UUID) (string, error) { + var status pillartypes.DomainStatus + if err := ReadPublication(d, "domainmgr", false, appUUID.String(), &status); err != nil { + return "", fmt.Errorf("failed to read DomainStatus of %s: %w", appUUID, err) + } + if status.DomainName == "" { + return "", fmt.Errorf("DomainStatus of %s carries no domain name yet", appUUID) + } + return status.DomainName, nil +} + +// AppVCPUAffinities returns, per guest vCPU index, the host CPUs that vCPU is +// allowed to run on. +// +// The guest-vCPU-to-host-thread mapping comes from the hypervisor over QMP +// because it cannot be recovered from the outside: QEMU does not name its vCPU +// threads unless started with debug-threads=on, and a domain's thread group also +// contains vhost_task helper threads that are indistinguishable from vCPU +// threads by name or by flags. Scanning /proc can therefore show that some +// thread is pinned, but not which vCPU it serves. +func (d *EdgeDevice) AppVCPUAffinities(appUUID uuid.UUID) (map[int][]uint32, error) { + domainName, err := d.AppDomainName(appUUID) + if err != nil { + return nil, err + } + vcpus, err := d.QueryVCPUs(domainName) + if err != nil { + return nil, err + } + affinities := make(map[int][]uint32, len(vcpus)) + for _, vcpu := range vcpus { + allowed, err := d.ThreadAffinity(vcpu.ThreadID) + if err != nil { + return nil, err + } + affinities[vcpu.CPUIndex] = allowed + } + return affinities, nil +} + +// AppCPUSet returns the logical CPUs of the cpuset an application is confined +// to, which bounds every thread of the workload including ones spawned later. +// A pinned workload's vCPU threads are additionally pinned individually within +// this set. +func (d *EdgeDevice) AppCPUSet(appUUID uuid.UUID) ([]uint32, error) { + domainName, err := d.AppDomainName(appUUID) + if err != nil { + return nil, err + } + // cgroup v1 and v2 place the file differently, and EVE has used both, so + // search rather than hard-code a layout. + script := fmt.Sprintf( + `find /sys/fs/cgroup -path '*%s*' -name 'cpuset.cpus' 2>/dev/null | while read -r f; do cat "$f"; break; done`, + domainName) + stdout, stderr, err := d.RunShellScript(script, hostQueryTimeout, 0) + if err != nil { + return nil, fmt.Errorf("failed to read cpuset of %s: %w (stderr: %s)", + domainName, err, stderr) + } + cpus := ParseCPUList(stdout) + if len(cpus) == 0 { + return nil, fmt.Errorf("no cpuset found for domain %s", domainName) + } + return cpus, nil +} + +// readCPUListFile reads a sysfs file holding a kernel CPU list. A missing file +// yields an empty list rather than an error: the kernel omits some of these +// entirely when the corresponding feature is not in use. +func (d *EdgeDevice) readCPUListFile(path string) ([]uint32, error) { + script := fmt.Sprintf("cat %s 2>/dev/null || true", path) + stdout, stderr, err := d.RunShellScript(script, hostQueryTimeout, 0) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w (stderr: %s)", path, err, stderr) + } + return ParseCPUList(stdout), nil +} + +// ParseCPUList expands a kernel CPU list such as "2", "0-3" or "0-2,5,7-8" into +// individual logical CPU ids. This is the format the kernel uses throughout +// sysfs and /proc for CPU sets. +func ParseCPUList(list string) []uint32 { + var cpus []uint32 + for _, part := range strings.Split(strings.TrimSpace(list), ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + loField, hiField, isRange := strings.Cut(part, "-") + lo, err := strconv.ParseUint(strings.TrimSpace(loField), 10, 32) + if err != nil { + continue + } + hi := lo + if isRange { + parsed, err := strconv.ParseUint(strings.TrimSpace(hiField), 10, 32) + if err != nil { + continue + } + hi = parsed + } + for cpu := lo; cpu <= hi; cpu++ { + cpus = append(cpus, uint32(cpu)) + } + } + return cpus +} + +func sortCPUs(cpus []uint32) { + for i := 0; i < len(cpus); i++ { + for j := i + 1; j < len(cpus); j++ { + if cpus[j] < cpus[i] { + cpus[i], cpus[j] = cpus[j], cpus[i] + } + } + } +} diff --git a/evetest/qmp.go b/evetest/qmp.go new file mode 100644 index 00000000000..05291ab3cec --- /dev/null +++ b/evetest/qmp.go @@ -0,0 +1,142 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package evetest + +import ( + "encoding/json" + "fmt" + "time" +) + +// qmpTimeout bounds a whole QMP exchange: connect, handshake and one command. +const qmpTimeout = 20 * time.Second + +// QMPCommand executes a QMP (QEMU Machine Protocol) command against the monitor +// of a running domain and returns the raw payload of the reply's "return" member. +// +// The monitor is a unix socket on the device, reached by tunnelling a +// direct-streamlocal channel over SSH, so nothing has to be installed on the +// device to use this. +// +// QMP is the authoritative source for facts about a running domain that cannot +// be recovered from the outside. The motivating example is which host thread +// serves which guest vCPU: QEMU only names its vCPU threads when started with +// debug-threads=on, and a domain's thread group also contains vhost_task helper +// threads that are indistinguishable from vCPU threads by name or by flags. So +// scanning /proc can show that some thread is pinned, but not which vCPU it +// belongs to; "query-cpus-fast" answers that directly. +// +// Note that a QEMU monitor socket serves one client at a time, and EVE itself +// connects to it briefly for its own operations (for example to pin vCPUs at +// domain start, or to change a VNC password). Calls here are short-lived, but a +// caller that polls should tolerate an occasional failure to connect rather +// than treat the first one as fatal. +func (d *EdgeDevice) QMPCommand(domainName, command string) (json.RawMessage, error) { + socket := fmt.Sprintf("/run/hypervisor/kvm/%s/qmp", domainName) + conn, err := d.DialViaSSH("unix", socket) + if err != nil { + return nil, fmt.Errorf("failed to reach QMP socket %s: %w", socket, err) + } + defer func() { _ = conn.Close() }() + + // An SSH channel does not support deadlines -- x/crypto/ssh rejects + // SetDeadline with "deadline not supported" -- so the exchange is bounded by + // closing the connection from a timer instead, which makes a blocked read + // fail immediately. Without this a hypervisor that never answers would hang + // the caller indefinitely. + finished := make(chan struct{}) + defer close(finished) + go func() { + select { + case <-time.After(qmpTimeout): + _ = conn.Close() + case <-finished: + } + }() + + dec := json.NewDecoder(conn) + + // QEMU announces itself first, and refuses commands until capabilities + // negotiation completes. + var greeting struct { + QMP json.RawMessage `json:"QMP"` + } + if err := dec.Decode(&greeting); err != nil { + return nil, fmt.Errorf("failed to read QMP greeting: %w", err) + } + if greeting.QMP == nil { + return nil, fmt.Errorf("unexpected first QMP message: not a greeting") + } + if _, err := qmpExecute(conn, dec, "qmp_capabilities"); err != nil { + return nil, fmt.Errorf("QMP capabilities negotiation failed: %w", err) + } + return qmpExecute(conn, dec, command) +} + +// qmpExecute sends one command and returns its reply payload, skipping any +// asynchronous events QEMU emits in between. +func qmpExecute(conn interface{ Write([]byte) (int, error) }, + dec *json.Decoder, command string) (json.RawMessage, error) { + request, err := json.Marshal(map[string]string{"execute": command}) + if err != nil { + return nil, err + } + if _, err := conn.Write(request); err != nil { + return nil, fmt.Errorf("failed to send QMP command %q: %w", command, err) + } + + for { + var reply struct { + Return json.RawMessage `json:"return"` + Error *struct { + Class string `json:"class"` + Desc string `json:"desc"` + } `json:"error"` + Event string `json:"event"` + } + if err := dec.Decode(&reply); err != nil { + return nil, fmt.Errorf("failed to read reply to QMP command %q: %w", + command, err) + } + if reply.Event != "" { + continue // asynchronous event, not our reply + } + if reply.Error != nil { + return nil, fmt.Errorf("QMP command %q failed: %s: %s", + command, reply.Error.Class, reply.Error.Desc) + } + return reply.Return, nil + } +} + +// QMPVCPU describes one guest vCPU as reported by the QMP "query-cpus-fast" +// command. ThreadID is the host thread serving that vCPU, which is what makes a +// vCPU's CPU affinity checkable in /proc. +type QMPVCPU struct { + CPUIndex int `json:"cpu-index"` + ThreadID int `json:"thread-id"` + Target string `json:"target"` +} + +// QueryVCPUs returns the guest vCPUs of a domain and the host thread serving +// each, ordered by guest vCPU index. +func (d *EdgeDevice) QueryVCPUs(domainName string) ([]QMPVCPU, error) { + payload, err := d.QMPCommand(domainName, "query-cpus-fast") + if err != nil { + return nil, err + } + var vcpus []QMPVCPU + if err := json.Unmarshal(payload, &vcpus); err != nil { + return nil, fmt.Errorf("failed to parse query-cpus-fast reply: %w", err) + } + // QEMU is not required to report them in index order. + for i := 0; i < len(vcpus); i++ { + for j := i + 1; j < len(vcpus); j++ { + if vcpus[j].CPUIndex < vcpus[i].CPUIndex { + vcpus[i], vcpus[j] = vcpus[j], vcpus[i] + } + } + } + return vcpus, nil +} diff --git a/evetest/requirements.go b/evetest/requirements.go index 742c724f1a0..0397f2d0f27 100644 --- a/evetest/requirements.go +++ b/evetest/requirements.go @@ -183,6 +183,17 @@ type RequireEdgeDevice struct { MinRAMInMiB uint32 // Default will be 8 GiB (8192 MiB). MinDiskSizeInMiB uint32 // Default will be 64 GiB (65536 MiB). + // ThreadsPerCore requests that the device's CPUs be presented as SMT + // hardware threads: the device gets MinCPUs logical CPUs arranged as + // MinCPUs/ThreadsPerCore physical cores with this many threads each. + // + // Zero (the default) presents every CPU as its own single-thread core, + // which is what every test that does not care about SMT gets. Set this to + // 2 only when the test needs sibling threads to exist -- for example to + // exercise CPU placement that allocates whole physical cores. MinCPUs must + // then be a multiple of it. + ThreadsPerCore uint8 + WithEVEVersion string WithHypervisor Hypervisor WithTPM bool diff --git a/evetest/setup.go b/evetest/setup.go index 89f428cd7c9..e37854e82c6 100644 --- a/evetest/setup.go +++ b/evetest/setup.go @@ -692,14 +692,20 @@ func (th *TestHarness) setupEVEDevices( }) } } + threadsPerCore := dev.requirement.ThreadsPerCore + if threadsPerCore > 1 && cpus%threadsPerCore != 0 { + th.t.Fatalf("Device %q requests %d threads per core, which does not "+ + "divide its %d CPUs", dev.requirement.Name, threadsPerCore, cpus) + } dev.spec = &api.EVEDevice{ - DeviceName: dev.requirement.Name, - Cpus: uint32(cpus), - MemoryBytes: uint64(memSizeInMiB) << 20, - SerialNumber: dev.serial, - WithTpm: dev.requirement.WithTPM, - Image: dev.imageRef, - Interfaces: interfaces, + DeviceName: dev.requirement.Name, + Cpus: uint32(cpus), + MemoryBytes: uint64(memSizeInMiB) << 20, + SerialNumber: dev.serial, + WithTpm: dev.requirement.WithTPM, + Image: dev.imageRef, + Interfaces: interfaces, + ThreadsPerCore: uint32(threadsPerCore), } setupReq.Devices = append(setupReq.Devices, dev.spec) } @@ -1282,6 +1288,7 @@ func (th *TestHarness) maybeReuseDevices( // Cannot reuse device if requirements changed. prevReq := dev.requirement equalReqs := newReq.MinCPUs == prevReq.MinCPUs && + newReq.ThreadsPerCore == prevReq.ThreadsPerCore && newReq.MinRAMInMiB == prevReq.MinRAMInMiB && newReq.MinDiskSizeInMiB == prevReq.MinDiskSizeInMiB && newReq.WithEVEVersion == prevReq.WithEVEVersion && From f99737a4eb74cc1560138bd319ad029f9365d312 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:05:53 +0000 Subject: [PATCH 10/15] evetest: e2e tests for dedicated CPU placement Covers the three placement shapes a controller can ask for, each asserted against the host's real topology rather than against expected CPU numbers. One-per-core: as many distinct physical cores as vCPUs, every vCPU pinned to exactly one host CPU, no two vCPUs sharing a core. Whole-core-SMT: both siblings of each core become vCPUs, and the guest's own view of its topology matches how it was actually pinned -- a guest told it has siblings that are not siblings will co-schedule work that then contends. The multi-app case is the one that catches interference: a whole-core-SMT app, a one-per-core app and a best-effort app deployed together must land on disjoint CPUs and disjoint physical cores, with housekeeping CPUs still available to the system. A test on any single app in isolation would pass while the allocator handed the same core to two workloads. Each reachable app needs its own forwarded edge-node port, since the port belongs to the node. Signed-off-by: Mikhail Malyshev --- evetest/tests/apps/cpuplacement_test.go | 989 ++++++++++++++++++++++++ evetest/tests/apps/helpers_test.go | 11 +- evetest/tests/apps/testsuite_test.go | 18 + 3 files changed, 1017 insertions(+), 1 deletion(-) create mode 100644 evetest/tests/apps/cpuplacement_test.go diff --git a/evetest/tests/apps/cpuplacement_test.go b/evetest/tests/apps/cpuplacement_test.go new file mode 100644 index 00000000000..46c4bb1218a --- /dev/null +++ b/evetest/tests/apps/cpuplacement_test.go @@ -0,0 +1,989 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test topology-aware CPU placement driven by the controller's per-app policy. + +package apps_test + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" +) + +// Timeouts shared by every phase of a placement scenario. +const ( + // placementRunningTimeout excludes the image download, which the framework + // waits for separately. + placementRunningTimeout = 10 * time.Minute + // placementSettleTimeout bounds how long the device may take to converge on + // the placement it reports -- including redistributing a best-effort app's + // cpuset once a pinned app takes its CPUs. + placementSettleTimeout = 2 * time.Minute + placementShellTimeout = 30 * time.Second + placementPolling = 5 * time.Second + // placementDevInfoInterval is the periodic device-info publish interval (in + // seconds) the tests that assert on the node's CPU pool report lower the + // device to. That report rides on ZInfoDevice and a change to it is not + // itself a publish trigger, so without this it would follow a workload + // starting or stopping only within the 10 minute default. 30s is the lowest + // value EVE accepts. + placementDevInfoInterval = 30 + // placementPoolReportTimeout bounds waiting for the node's CPU pool report to + // catch up with the workloads currently running. It covers several publish + // intervals plus the trip to the controller. + placementPoolReportTimeout = 4 * time.Minute +) + +// Placement quality as it appears on the wire. DomainStatus.PlacementQuality is +// a Go uint8 enum with no custom JSON marshaller, so it serializes as a number +// and has to be compared as one. +const ( + placementQualityUnspecified = 0 + placementQualityOptimal = 1 + placementQualityNeedsRepack = 2 +) + +// runCPUPlacementScenario deploys a whole set of applications with different CPU +// placement policies onto one device and verifies both what each application got +// and how their allocations relate to one another. +// +// Contention is the point. A single application's allocation can look correct by +// accident -- with nothing else on the machine, almost any set of CPUs satisfies +// it -- while the properties the allocator actually exists to guarantee (no host +// CPU dedicated twice, no physical core shared by two whole-core workloads, EVE +// and best-effort workloads left somewhere to run) only have meaning once +// several workloads compete. Booting a device costs minutes, so a scenario is a +// table: every application in it is deployed on the same boot, checked +// individually, and then the set is checked as a whole. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- placement is not network dependent; a +// single mgmt+apps port is enough to run the apps and reach them over SSH. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps). +// - Local NI "local-ni" on ethernet0. +// - One container app (lfedge/evetest-ubuntu-ctr) per table entry, each with +// its own forwarded SSH port and its own CPU placement policy. +// - MinCPUs/ThreadsPerCore give the device VM enough physical cores that the +// table fits *and* EVE keeps a non-empty housekeeping set; an allocation +// that would empty it is refused, so an over-subscribed table shows up as a +// deployment failure rather than as a wrong placement. +// +// Phases / assertions +// ------------------- +// 1. apps-running: every app reaches RUNNING, i.e. the policies are jointly +// satisfiable and do not wedge deployment. +// 2. per app (see assertAppPlacement): the allocation domainmgr recorded has +// the shape the policy asked for, the kernel enforces it vCPU by vCPU, the +// assigned CPUs relate to physical cores the way the mode promises, and the +// guest sees exactly the vCPUs it was configured with. +// 3. the set as a whole (see assertPlacementSetInvariants): dedicated sets are +// disjoint, no two whole-core apps share a physical core, housekeeping is +// non-empty, and a best-effort app is not confined to somebody else's +// dedicated CPUs. +// +// Phase 2 reads what EVE decided and then checks it against ground truth via +// /proc//task, /sys/devices/system/cpu and QMP -- standard interfaces that +// stay valid across EVE versions. EVE's claim alone would not be evidence, and +// the kernel state alone would not show whether the policy or something else +// produced it: the test needs both, and that they agree. +// +// Test params +// ----------- +// - HYPERVISOR. Skipped under Kubevirt, where concrete CPU selection belongs +// to the kubelet rather than to the pillar allocator this test exercises. +// +// Suite placement +// --------------- +// - TestAppsSuite (deploys apps, hence hypervisor-parameterized). +func runCPUPlacementScenario(test *testing.T, sc cpuPlacementScenario) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + if hypervisor == evetest.HypervisorKubevirt { + evetestT.Skip("under Kubevirt the kubelet selects CPUs, not the pillar allocator") + } + + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + MinCPUs: sc.minCPUs, + ThreadsPerCore: sc.deviceThreadsPerCore, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := addLocalNI(devConfig) + + evetest.Logger().Infof("CPU placement scenario %q: %d application(s)", + sc.name, len(sc.apps)) + deployed := make([]*placedApp, 0, len(sc.apps)) + for _, spec := range sc.apps { + appUUID := devConfig.AddApplication(placementAppConfig(spec, niUUID, 0)) + deployed = append(deployed, &placedApp{spec: spec, uuid: appUUID}) + } + device.ApplyConfig(devConfig, true, true) + evetest.Checkpoint("config-applied") + + // Phase 1: the policies must be jointly satisfiable, not wedge deployment. + for _, app := range deployed { + device.WaitUntilAppIsRunning(app.uuid, placementRunningTimeout) + } + evetest.Checkpoint("apps-running") + + // Dump how the device actually placed everything before asserting anything. + // An end-to-end run is expensive, so a failure has to be diagnosable from + // the log it already produced rather than from a second run with more + // prints. + logPlacementDiagnostics(device, appUUIDs(deployed), placementShellTimeout) + + topo, err := device.HostCPUTopology() + t.Expect(err).ToNot(HaveOccurred()) + + // Phase 2: each application on its own. Nothing was running before this + // scenario, so every workload must have landed on its planned slot. + for _, app := range deployed { + assertAppPlacement(t, device, topo, app, true) + evetest.Checkpoint("placement-verified-" + app.spec.appName) + } + + // Phase 3: the set as a whole. + assertPlacementSetInvariants(t, device, topo, deployed) + evetest.Checkpoint("set-invariants-verified") + + for _, app := range deployed { + deleteAppAndWait(t, device, devConfig, app.uuid) + } +} + +// assertAppPlacement verifies one application's placement: the allocation +// domainmgr recorded, that the kernel enforces it per guest vCPU, the +// relationship the assigned CPUs have to physical cores, and what the guest +// sees. It stores what it read on the placedApp, so the set-wide invariants can +// be checked afterwards without a second round of device queries. +// +// requireOptimal additionally demands that the workload landed on the slot the +// batch plan set aside for it. That holds whenever the pinned workloads either +// all start from nothing or start into a set that is already placed as the plan +// intended, which is the case for every scenario here. It does not hold once a +// pinned workload has been placed while others were stopped -- it was then +// planned as if it were alone on the node, and a workload starting afterwards +// can only take what is still free -- so a caller exercising that must pass +// false and check validity alone. +func assertAppPlacement(t *GomegaWithT, device *evetest.EdgeDevice, + topo evetest.HostTopology, app *placedApp, requireOptimal bool) { + spec := app.spec + + // The allocation domainmgr recorded. This is EVE's claim, not the proof -- + // the proof is in the affinity and topology checks below. + t.Eventually(func(g Gomega) { + status, err := readDomainCPUStatus(device, app.uuid) + g.Expect(err).ToNot(HaveOccurred()) + if spec.wantPinned { + g.Expect(status.OrderedCPUs).ToNot(BeEmpty(), + "domainmgr recorded no per-vCPU CPU assignment for %s", spec.appName) + } else { + g.Expect(status.CPUs).ToNot(BeEmpty(), + "domainmgr recorded no CPU set at all for %s", spec.appName) + } + app.status = status + }, placementSettleTimeout, placementPolling).Should(Succeed()) + + status := app.status + // EmulatorCPUs is the shared housekeeping pool, which io_placement + // "housekeeping" also appends to CPUs. Subtracting it leaves the CPUs this + // workload alone may use, which is what the set-wide invariants are about. + app.dedicated = subtractCPUs(status.CPUs, status.EmulatorCPUs) + + evetest.Logger().Infof("placement recorded for %q: vCPU CPUs %v, dedicated set %v, "+ + "emulator CPUs %v, guest topology %d/%d/%d, pinned=%v, quality=%s", + spec.appName, status.OrderedCPUs, status.CPUs, status.EmulatorCPUs, + status.VMTopology.Sockets, status.VMTopology.Cores, status.VMTopology.Threads, + status.CPUsPinned, placementQualityName(status.PlacementQuality)) + + if !spec.wantPinned { + assertBestEffortPlacement(t, device, app) + return + } + + assigned := status.OrderedCPUs + // A dedicated policy has to pin on its own: the app never sets the legacy + // pin_cpu flag, so this also proves the policy is what drove pinning. + t.Expect(status.CPUsPinned).To(BeTrue(), + "a dedicated CPU policy must imply pinning without the legacy pin_cpu flag") + t.Expect(assigned).To(HaveLen(spec.vCPUs), "one host CPU must be assigned per vCPU") + t.Expect(uniqueCPUs(assigned)).To(HaveLen(spec.vCPUs), + "the same host CPU must not back two vCPUs (assigned: %v)", assigned) + t.Expect(status.VMTopology.Threads).To(Equal(spec.wantGuestThreads), + "%s must advertise a guest topology with %d thread(s) per core", + spec.mode, spec.wantGuestThreads) + t.Expect(status.VMTopology.Cores).To(Equal(spec.vCPUs/spec.wantGuestThreads), + "%s must expose %d guest core(s)", spec.mode, spec.vCPUs/spec.wantGuestThreads) + for _, cpu := range assigned { + t.Expect(status.CPUs).To(ContainElement(cpu), + "host CPU %d backs a vCPU but is missing from the dedicated set %v", + cpu, status.CPUs) + } + // The table fits the machine, so every pinned workload must have landed on + // its planned slot -- or on an equally good one. "needs-repack" here would + // mean the plan and the allocation disagree even though nothing had to be + // worked around, i.e. the ordering that makes placement independent of + // activation order did not hold. + if requireOptimal { + t.Expect(status.PlacementQuality).To(Equal(placementQualityOptimal), + "%q reports placement quality %s although every pinned workload was "+ + "placed against the full set; a repack should never be needed when "+ + "nothing had to be worked around", + spec.appName, placementQualityName(status.PlacementQuality)) + } + + // The kernel agrees, per guest vCPU. The framework resolves the + // guest-vCPU-to-host-thread mapping over QMP, which is the only place it + // exists (see EdgeDevice.AppVCPUAffinities). + var affinities map[int][]uint32 + t.Eventually(func(g Gomega) { + var err error + affinities, err = device.AppVCPUAffinities(app.uuid) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(affinities).To(HaveLen(spec.vCPUs)) + }, placementSettleTimeout, placementPolling).Should(Succeed()) + + evetest.Logger().Infof("guest vCPU affinities for %q: %v", spec.appName, affinities) + for vcpu, cpu := range assigned { + t.Expect(affinities[vcpu]).To(ConsistOf(cpu), + "guest vCPU %d of %q must be pinned to exactly host CPU %d, but may run "+ + "on %v; a cgroup cpuset alone would leave it free to float across "+ + "the whole dedicated set", vcpu, spec.appName, cpu, affinities[vcpu]) + } + + // The physical-core relationship the mode promises. + assertCoreRelationship(t, topo, assigned, spec) + + // The cpuset confines every thread of the workload, including any spawned + // later, and must cover the dedicated CPUs. + cpuset, err := device.AppCPUSet(app.uuid) + t.Expect(err).ToNot(HaveOccurred()) + app.cpuset = cpuset + evetest.Logger().Infof("cpuset for %q: %v", spec.appName, cpuset) + t.Expect(cpuset).To(ContainElements(intsOf(assigned)), + "the application cpuset %v must cover its dedicated CPUs %v", cpuset, assigned) + + assertGuestCPUCount(t, device, app) +} + +// assertBestEffortPlacement verifies an application that asked for shared +// (best-effort) placement. Such a workload must be left alone: giving it a +// dedicated set, a synthesized SMT topology or per-vCPU pinning would all be +// wrong, and reporting a placement quality for it would claim an evaluation +// that never happened. +func assertBestEffortPlacement(t *GomegaWithT, device *evetest.EdgeDevice, + app *placedApp) { + spec := app.spec + status := app.status + + t.Expect(status.CPUsPinned).To(BeFalse(), + "%q asked for shared placement and must not be pinned", spec.appName) + t.Expect(status.OrderedCPUs).To(BeEmpty(), + "%q asked for shared placement, so no vCPU may be bound to a fixed host CPU "+ + "(recorded: %v)", spec.appName, status.OrderedCPUs) + t.Expect(status.VMTopology.Threads).To(BeZero(), + "%q asked for shared placement, so it must keep the flat guest topology; "+ + "a synthesized threads=%d would tell the guest its vCPUs are SMT "+ + "siblings when nothing guarantees they are", + spec.appName, status.VMTopology.Threads) + t.Expect(status.PlacementQuality).To(Equal(placementQualityUnspecified), + "%q is not whole-core pinned, so its placement quality was never "+ + "evaluated and must not be reported as %s", + spec.appName, placementQualityName(status.PlacementQuality)) + + // Every vCPU stays free to float across the whole cpuset. Read the cpuset + // and the affinities together so they cannot describe different moments. + t.Eventually(func(g Gomega) { + cpuset, err := device.AppCPUSet(app.uuid) + g.Expect(err).ToNot(HaveOccurred()) + affinities, err := device.AppVCPUAffinities(app.uuid) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(affinities).To(HaveLen(spec.vCPUs)) + for vcpu, allowed := range affinities { + g.Expect(allowed).To(ConsistOf(intsOf(cpuset)), + "vCPU %d of best-effort app %q may only run on %v while its cpuset "+ + "is %v; a best-effort workload must not be pinned to a subset "+ + "of its own cpuset", vcpu, spec.appName, allowed, cpuset) + } + app.cpuset = cpuset + }, placementSettleTimeout, placementPolling).Should(Succeed()) + evetest.Logger().Infof("cpuset for best-effort app %q: %v", spec.appName, app.cpuset) + + assertGuestCPUCount(t, device, app) +} + +// assertGuestCPUCount checks that the guest was told how many CPUs it has and +// sees exactly those -- a placement that quietly handed the guest a different +// vCPU count would satisfy every host-side check. +func assertGuestCPUCount(t *GomegaWithT, device *evetest.EdgeDevice, app *placedApp) { + t.Eventually(func(g Gomega) { + stdout, _, err := device.RunShellScriptInsideApp(app.uuid, appAuth, + "nproc", placementShellTimeout, 0) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(strings.TrimSpace(stdout)).To(Equal(strconv.Itoa(app.spec.vCPUs)), + "%q must see exactly the vCPUs it was configured with", app.spec.appName) + }, placementSettleTimeout, placementPolling).Should(Succeed()) +} + +// assertPlacementSetInvariants checks the properties that only exist once +// several workloads share a machine. They are what makes a dedicated CPU +// actually dedicated: each of them can be violated by an allocator that places +// every workload perfectly when considered on its own. +func assertPlacementSetInvariants(t *GomegaWithT, device *evetest.EdgeDevice, + topo evetest.HostTopology, deployed []*placedApp) { + var pinned []*placedApp + for _, app := range deployed { + if app.spec.wantPinned { + pinned = append(pinned, app) + } + } + + // 1. Dedicated sets are disjoint. Parked SMT siblings count: they are held + // back precisely so nobody else runs on the core, so handing one to another + // workload defeats the point of asking for whole cores. + owner := map[uint32]string{} + for _, app := range pinned { + for _, cpu := range app.dedicated { + holder, taken := owner[cpu] + t.Expect(taken).To(BeFalse(), + "host CPU %d is dedicated to both %q and %q; two workloads with "+ + "dedicated CPUs must never overlap (%q holds %v, %q holds %v)", + cpu, holder, app.spec.appName, holder, dedicatedOf(pinned, holder), + app.spec.appName, app.dedicated) + owner[cpu] = app.spec.appName + } + } + + // The kernel-side counterpart of the above: bookkeeping that keeps the sets + // apart is worthless if the cgroup cpuset still lets one workload's threads + // run on another's dedicated CPUs. + for _, app := range pinned { + for _, cpu := range app.cpuset { + holder, dedicated := owner[cpu] + t.Expect(dedicated && holder != app.spec.appName).To(BeFalse(), + "the cpuset %v of %q includes host CPU %d, which is dedicated to %q", + app.cpuset, app.spec.appName, cpu, holder) + } + } + + // 2. No physical core shared between two whole-core workloads. Disjoint CPU + // sets are not enough: two workloads landing on the two SMT siblings of one + // core hold disjoint sets and still contend for the same core's execution + // resources, which is exactly what full_pcpus_only is bought to prevent. + for i, a := range pinned { + if a.spec.coreRule == coreRuleUnconstrained { + continue + } + for _, b := range pinned[i+1:] { + if b.spec.coreRule == coreRuleUnconstrained { + continue + } + for _, ca := range a.status.OrderedCPUs { + for _, cb := range b.status.OrderedCPUs { + t.Expect(topo.SameCore(ca, cb)).To(BeFalse(), + "host CPUs %d (%q) and %d (%q) are SMT siblings on one "+ + "physical core, but both workloads asked for whole "+ + "cores (%q: %v, %q: %v, siblings of %d: %v)", + ca, a.spec.appName, cb, b.spec.appName, + a.spec.appName, a.status.OrderedCPUs, + b.spec.appName, b.status.OrderedCPUs, ca, topo.SiblingsOf(ca)) + } + } + } + } + + // 3. Housekeeping is not empty. EVE's own services, the emulator threads and + // every best-effort workload live on whatever is left, so an allocation that + // consumes the last online CPU takes the device down with it. + // + // Retried, unlike the placement reads above: the set of online CPUs is a + // kernel fact that cannot change while a test runs, so the only way this read + // fails is the transport -- an SSH session that did not come up. Failing a + // placement verdict on that would report a bug that is not there, and these + // tests issue a burst of short-lived SSH sessions right here. + var online []uint32 + t.Eventually(func(g Gomega) { + var err error + online, err = device.OnlineCPUs() + g.Expect(err).ToNot(HaveOccurred()) + }, placementSettleTimeout, placementPolling).Should(Succeed()) + var housekeeping []uint32 + for _, cpu := range online { + if _, dedicated := owner[cpu]; !dedicated { + housekeeping = append(housekeeping, cpu) + } + } + evetest.Logger().Infof("online CPUs %v, dedicated %v, housekeeping %v", + online, sortedKeys(owner), housekeeping) + t.Expect(housekeeping).ToNot(BeEmpty(), + "every online CPU (%v) ended up dedicated to a workload, leaving EVE and "+ + "any best-effort workload nowhere to run", online) + + // 4. A best-effort workload is not confined to somebody else's dedicated + // CPUs. Eventually, because such a workload may have been deployed before + // the pinned ones: it starts out with the whole machine in its cpuset and is + // only pushed off as the pinned workloads take their CPUs. That + // redistribution is the step this checks -- without it a best-effort app + // keeps running on cores that are supposed to be exclusive. + for _, app := range deployed { + if app.spec.wantPinned { + continue + } + t.Eventually(func(g Gomega) { + cpuset, err := device.AppCPUSet(app.uuid) + g.Expect(err).ToNot(HaveOccurred()) + for _, cpu := range cpuset { + holder, dedicated := owner[cpu] + g.Expect(dedicated).To(BeFalse(), + "the cpuset %v of best-effort app %q includes host CPU %d, "+ + "which is dedicated to %q; best-effort workloads belong in "+ + "the housekeeping set %v", cpuset, app.spec.appName, cpu, + holder, housekeeping) + } + app.cpuset = cpuset + }, placementSettleTimeout, placementPolling).Should(Succeed()) + evetest.Logger().Infof("best-effort app %q settled on cpuset %v", + app.spec.appName, app.cpuset) + } +} + +// coreRule is the relationship an application's assigned host CPUs must have to +// physical cores. It is the one property a count- or set-based assertion cannot +// capture, and the only thing that distinguishes the whole-core modes from each +// other and from thread-granular placement. +type coreRule int + +const ( + // coreRuleUnconstrained places no requirement on cores: the workload is + // shared, or dedicated at SMT-thread granularity (no full_pcpus_only). + coreRuleUnconstrained coreRule = iota + // coreRuleDistinctCores requires a physical core of its own per vCPU, with + // the sibling threads held back rather than handed to another workload. + coreRuleDistinctCores + // coreRuleSiblingPairs requires vCPUs 2k and 2k+1 to land on the two SMT + // siblings of one physical core, as the guest is told they are. + coreRuleSiblingPairs +) + +// cpuPlacementApp is one application in a placement scenario: the policy it +// asks for and the outcome that policy must produce. +type cpuPlacementApp struct { + // mode describes the placement mode, for assertion messages. + mode string + // appName is the deployed application's display name. + appName string + // vCPUs is the vCPU count requested by the application. + vCPUs int + // sshFwdPort is the edge-node port forwarded to this app's sshd. It lives in + // the device's namespace, so every app in a scenario needs its own. + sshFwdPort uint16 + // placement is the CPU placement intent sent by the controller. + placement evetest.CPUPlacementConfig + // wantPinned is whether the policy must result in the workload getting host + // CPUs of its own. + wantPinned bool + // wantGuestThreads is the threads-per-core the guest must be shown. Only + // meaningful for a pinned app; a shared one keeps the flat topology. + wantGuestThreads int + // coreRule is the relationship the assigned CPUs must have to physical + // cores. + coreRule coreRule +} + +// cpuPlacementScenario is one set of applications to deploy together, plus what +// the device VM must provide for the set to fit. Modes and combinations differ +// only in these values, so they share one body rather than being copied. +type cpuPlacementScenario struct { + // name describes the scenario, for the log. + name string + // deviceThreadsPerCore is the SMT topology the device VM itself needs. A + // mode that consumes both siblings of a core cannot be tested on a device + // whose CPUs are all single-thread cores: there is no sibling to take. + deviceThreadsPerCore uint8 + // minCPUs must cover every app's cores and still leave EVE a non-empty + // housekeeping set; an allocation that would empty it is refused. + minCPUs uint8 + // apps are deployed together, in this order, on one device boot. + apps []cpuPlacementApp +} + +// placedApp is a deployed application plus everything the assertions read about +// it, so the set-wide invariants can be checked without querying the device +// again for state the per-app phase already has. +type placedApp struct { + spec cpuPlacementApp + uuid uuid.UUID + status domainCPUStatus + // dedicated is the host CPUs this workload alone may use (vCPU CPUs plus + // parked siblings), i.e. its recorded CPU set minus the shared emulator + // pool. Empty for a best-effort workload. + dedicated []uint32 + // cpuset is the cgroup cpuset the workload is confined to. + cpuset []uint32 +} + +// onePerCoreApp asks for a dedicated physical core per vCPU, with the sibling +// threads parked idle. +func onePerCoreApp(appName string, vCPUs int, sshFwdPort uint16) cpuPlacementApp { + return cpuPlacementApp{ + mode: "one-per-core", + appName: appName, + vCPUs: vCPUs, + sshFwdPort: sshFwdPort, + placement: evetest.CPUPlacementConfig{ + Policy: eveconfig.CpuPolicy_CPU_POLICY_DEDICATED, + FullPCPUsOnly: true, + ThreadsPerCore: 1, + NUMAPolicy: eveconfig.NumaPolicy_NUMA_POLICY_BEST_EFFORT, + }, + wantPinned: true, + wantGuestThreads: 1, + coreRule: coreRuleDistinctCores, + } +} + +// wholeCoreSMTApp asks for dedicated whole physical cores with both SMT siblings +// of each becoming vCPUs, and the guest told which of its vCPUs are siblings. +func wholeCoreSMTApp(appName string, vCPUs int, sshFwdPort uint16) cpuPlacementApp { + return cpuPlacementApp{ + mode: "whole-core-smt", + appName: appName, + vCPUs: vCPUs, + sshFwdPort: sshFwdPort, + placement: evetest.CPUPlacementConfig{ + Policy: eveconfig.CpuPolicy_CPU_POLICY_DEDICATED, + FullPCPUsOnly: true, + ThreadsPerCore: 2, + NUMAPolicy: eveconfig.NumaPolicy_NUMA_POLICY_BEST_EFFORT, + }, + wantPinned: true, + wantGuestThreads: 2, + coreRule: coreRuleSiblingPairs, + } +} + +// sharedApp asks for best-effort placement in the shared pool, which is what +// every ordinary workload gets. It is in a placement scenario as the workload +// the dedicated ones must not disturb, and that must not disturb them. +func sharedApp(appName string, vCPUs int, sshFwdPort uint16) cpuPlacementApp { + return cpuPlacementApp{ + mode: "shared", + appName: appName, + vCPUs: vCPUs, + sshFwdPort: sshFwdPort, + placement: evetest.CPUPlacementConfig{ + Policy: eveconfig.CpuPolicy_CPU_POLICY_SHARED, + }, + wantPinned: false, + coreRule: coreRuleUnconstrained, + } +} + +// threadGranularApp asks for dedicated CPUs at SMT-thread granularity: +// CPU_POLICY_DEDICATED *without* full_pcpus_only. The workload gets host CPUs +// no other workload may use, but they are individual logical CPUs rather than +// whole physical cores, so it may well end up on a core whose other thread +// belongs to somebody else. That is the pre-policy pinning behaviour, and it is +// still what a workload wants when all it needs is not to be descheduled. +// +// It is the only shape that can leave a physical core *half* owned, which is why +// TestCPUPlacementNeedsRepack builds its fragmented node out of these: a +// whole-core workload takes and releases whole cores, so no arrangement of +// whole-core workloads can ever fragment the node. +// +// Its allocation is deliberately not asserted by assertAppPlacement: this path +// records no per-vCPU assignment (OrderedCPUs stays empty) and synthesizes no +// guest SMT topology, so it is checked by assertThreadGranularPlacement instead. +func threadGranularApp(appName string, vCPUs int, sshFwdPort uint16) cpuPlacementApp { + return cpuPlacementApp{ + mode: "thread-granular", + appName: appName, + vCPUs: vCPUs, + sshFwdPort: sshFwdPort, + placement: evetest.CPUPlacementConfig{ + Policy: eveconfig.CpuPolicy_CPU_POLICY_DEDICATED, + // The point of this constructor: dedicated CPUs without whole + // cores. ThreadsPerCore is left unset because it only means + // anything for whole-core placement. + FullPCPUsOnly: false, + NUMAPolicy: eveconfig.NumaPolicy_NUMA_POLICY_BEST_EFFORT, + }, + wantPinned: true, + wantGuestThreads: 0, + coreRule: coreRuleUnconstrained, + } +} + +// assertCoreRelationship checks how one application's assigned CPUs relate to +// physical cores, which is the whole point of full_pcpus_only. +func assertCoreRelationship(t *GomegaWithT, topo evetest.HostTopology, + assigned []uint32, spec cpuPlacementApp) { + switch spec.coreRule { + case coreRuleUnconstrained: + return + + case coreRuleDistinctCores: + for i, a := range assigned { + for j, b := range assigned { + if i >= j { + continue + } + t.Expect(topo.SameCore(a, b)).To(BeFalse(), + "host CPUs %d and %d are SMT siblings on one physical core, but "+ + "%s requires a distinct core per vCPU (assigned: %v, "+ + "siblings of %d: %v)", + a, b, spec.mode, assigned, a, topo.SiblingsOf(a)) + } + } + + case coreRuleSiblingPairs: + // The guest is told vCPUs 2k and 2k+1 are siblings, so they must really + // be siblings on the host. A placement that used complete sibling pairs + // but paired them up differently would still satisfy every count- and + // set-based check while lying to the guest about which vCPUs share a + // core -- which is exactly the property the mode exists to provide. + t.Expect(len(assigned)%2).To(Equal(0), + "whole-core-smt cannot assign an odd number of CPUs") + for i := 0; i+1 < len(assigned); i += 2 { + a, b := assigned[i], assigned[i+1] + t.Expect(topo.SameCore(a, b)).To(BeTrue(), + "guest vCPUs %d and %d are presented to the guest as SMT siblings, so "+ + "host CPUs %d and %d must be siblings on one physical core "+ + "(assigned: %v, siblings of %d: %v)", + i, i+1, a, b, assigned, a, topo.SiblingsOf(a)) + } + // Distinct pairs must not share a core, or the workload would have been + // given fewer physical cores than it asked for. + for i := 0; i+1 < len(assigned); i += 2 { + for j := i + 2; j+1 < len(assigned); j += 2 { + t.Expect(topo.SameCore(assigned[i], assigned[j])).To(BeFalse(), + "vCPU pairs (%d,%d) and (%d,%d) share one physical core; %s must "+ + "allocate a whole core per pair (assigned: %v)", + i, i+1, j, j+1, spec.mode, assigned) + } + } + } +} + +// TestCPUPlacementOnePerCore exercises the one-per-core mode: every vCPU gets a +// physical core of its own and the sibling threads are held back rather than +// handed to another workload. See runCPUPlacementScenario for the phases. +func TestCPUPlacementOnePerCore(test *testing.T) { + runCPUPlacementScenario(test, cpuPlacementScenario{ + name: "one-per-core", + // Two dedicated cores still leave EVE a non-empty housekeeping set. + minCPUs: 6, + // No SMT is needed: this mode uses one thread per core by definition, + // so the device's default single-thread cores are exactly right. + deviceThreadsPerCore: 0, + apps: []cpuPlacementApp{ + onePerCoreApp("cpu-pinned-app", 2, appSSHFwdPort), + }, + }) +} + +// TestCPUPlacementWholeCoreSMT exercises the whole-core-SMT mode: the workload +// gets whole physical cores and *both* SMT siblings of each become vCPUs, with +// the guest told which of its vCPUs are siblings so it can place its own hot +// work accordingly. This is the mode a poll-mode datapath wants, and the one +// where a mismatch between the advertised and the real sibling relationship +// silently costs throughput rather than failing outright. +// +// It requires an SMT topology on the device VM itself (deviceThreadsPerCore +// below): with single-thread cores there is no sibling to allocate a second vCPU +// from, and the allocator correctly refuses instead of quietly using two cores. +func TestCPUPlacementWholeCoreSMT(test *testing.T) { + runCPUPlacementScenario(test, cpuPlacementScenario{ + name: "whole-core-smt", + // 8 CPUs as 4 dual-thread cores: two go to the app, two are left for + // EVE's housekeeping set. + minCPUs: 8, + deviceThreadsPerCore: 2, + apps: []cpuPlacementApp{ + wholeCoreSMTApp("cpu-smt-pinned-app", 4, appSSHFwdPort), // two whole cores + }, + }) +} + +// TestCPUPlacementMultiApp deploys the three placement modes side by side on one +// device: a whole-core-SMT app, a one-per-core app and an ordinary best-effort +// app, all on the same boot. +// +// This is the case the single-mode tests cannot cover. With one workload on the +// machine there is nothing to collide with, so an allocator that ignores what +// other workloads hold still looks correct; here the two dedicated workloads +// must end up on disjoint CPUs *and* disjoint physical cores, and the +// best-effort one -- deployed first, so it starts out with the whole machine in +// its cpuset -- must be pushed off their cores as they take them. It also +// exercises the ordering that makes placement independent of activation order: +// whole-core-SMT is the most constrained mode (it can only use a core that +// really has two hardware threads), so if the machine were carved up in arrival +// order the flexible one-per-core app could take the cores it needs. +// +// The table is sized to the device: 8 CPUs as 4 dual-thread cores, of which EVE +// reserves the lowest (making its whole core unallocatable), leaves three +// allocatable cores -- one for the SMT app's two vCPUs, two for the one-per-core +// app's two vCPUs -- and the reserved core as housekeeping for EVE and the +// best-effort app. +func TestCPUPlacementMultiApp(test *testing.T) { + runCPUPlacementScenario(test, cpuPlacementScenario{ + name: "multi-app", + minCPUs: 8, + deviceThreadsPerCore: 2, + apps: []cpuPlacementApp{ + // First on purpose: a best-effort app deployed before any dedicated + // one starts with every CPU in its cpuset, so it is only off the + // dedicated cores if the device actively redistributes. + sharedApp("cpu-shared-app", 1, appSSHFwdPort+2), + wholeCoreSMTApp("cpu-smt-app", 2, appSSHFwdPort), // one whole core + onePerCoreApp("cpu-core-app", 2, appSSHFwdPort+1), // two whole cores + }, + }) +} + +// placementDiagnosticsScript dumps everything relevant to how the device placed +// the applications' threads. It deliberately filters as little as possible: the +// point is to show what is actually there, including the cases the assertions +// did not anticipate. +const placementDiagnosticsScript = `UUIDS='@UUIDS@' +echo "## kernel / cpu count" +uname -r; nproc + +echo "## per-CPU topology (core_id, package, siblings)" +for d in /sys/devices/system/cpu/cpu[0-9]*; do + printf ' cpu%s core_id=%s pkg=%s siblings=%s\n' "${d#*/cpu}" \ + "$(cat "$d/topology/core_id" 2>/dev/null)" \ + "$(cat "$d/topology/physical_package_id" 2>/dev/null)" \ + "$(cat "$d/topology/thread_siblings_list" 2>/dev/null)" +done + +echo "## online / isolated" +echo " online=$(cat /sys/devices/system/cpu/online 2>/dev/null)" +echo " isolated=$(cat /sys/devices/system/cpu/isolated 2>/dev/null)" + +echo "## processes whose cmdline mentions qemu or an app UUID" +for p in /proc/[0-9]*; do + cl=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null) + case "$cl" in + @UUID_CASES@) ;; + *) continue ;; + esac + echo " --- pid=${p#/proc/} comm=$(cat "$p/comm" 2>/dev/null)" + echo " cmdline=$(echo "$cl" | cut -c1-500)" + echo " cgroup=$(tr '\n' ' ' < "$p/cgroup" 2>/dev/null)" + for td in "$p"/task/*; do + echo " tid=${td##*/} comm=$(cat "$td/comm" 2>/dev/null) aff=$(awk '/Cpus_allowed_list/{print $2}' "$td/status" 2>/dev/null)" + done +done + +echo "## every process comm (in case the filter above matched nothing)" +for p in /proc/[0-9]*; do + echo " ${p#/proc/} $(cat "$p/comm" 2>/dev/null)" +done + +echo "## app cgroup cpusets" +for f in $(find /sys/fs/cgroup -name 'cpuset.cpus*' 2>/dev/null | grep -i @CGROUP_GREP@); do + echo " $f = $(cat "$f" 2>/dev/null)" +done + +echo "## the CPU plan domainmgr computed for the configured set" +cat /run/domainmgr/cpuplan.json 2>/dev/null + +echo "## DomainStatus as published by domainmgr" +for u in $UUIDS; do + cat "/run/domainmgr/DomainStatus/$u.json" 2>/dev/null | cut -c1-4000 + echo +done +` + +// logPlacementDiagnostics runs the diagnostics script and logs its output. It +// never fails the test: it exists so that whatever the assertions conclude, the +// evidence is in the log. +func logPlacementDiagnostics(device *evetest.EdgeDevice, appUUIDs []uuid.UUID, + timeout time.Duration) { + var uuids, cases, grep []string + cases = append(cases, "*qemu*") + grep = append(grep, "-e eve-user-apps") + for _, appUUID := range appUUIDs { + uuids = append(uuids, appUUID.String()) + cases = append(cases, "*"+appUUID.String()+"*") + grep = append(grep, "-e "+appUUID.String()) + } + script := placementDiagnosticsScript + script = strings.ReplaceAll(script, "@UUIDS@", strings.Join(uuids, " ")) + script = strings.ReplaceAll(script, "@UUID_CASES@", strings.Join(cases, "|")) + script = strings.ReplaceAll(script, "@CGROUP_GREP@", strings.Join(grep, " ")) + stdout, stderr, err := device.RunShellScript(script, timeout, 0) + if err != nil { + evetest.Logger().Warnf("CPU placement diagnostics failed: %v (stderr: %s)", + err, stderr) + } + evetest.Logger().Infof("CPU placement diagnostics for apps %v:\n%s", uuids, stdout) +} + +// domainCPUStatus is the subset of the DomainStatus that domainmgr publishes +// under /run which describes the CPU allocation. A local struct is used rather +// than pillar's own type because evetest builds against a released pillar +// module, which does not carry fields added on a feature branch. +type domainCPUStatus struct { + // DomainName identifies the domain on the device and therefore locates its + // QEMU monitor socket. + DomainName string + CPUs []uint32 + CPUsPinned bool + OrderedCPUs []uint32 + EmulatorCPUs []uint32 + VMTopology struct { + Sockets int + Cores int + Threads int + } + // PlacementQuality is pillar's CPUPlacementQuality, a uint8 enum with no + // custom JSON marshaller, so it arrives as a number rather than a name. + PlacementQuality int +} + +// readDomainCPUStatus reads back the allocation domainmgr decided on, so it can +// be cross-checked against what the kernel is actually enforcing. It is EVE's +// claim, not the proof. +func readDomainCPUStatus(device *evetest.EdgeDevice, + appUUID uuid.UUID) (domainCPUStatus, error) { + var status domainCPUStatus + path := fmt.Sprintf("/run/domainmgr/DomainStatus/%s.json", appUUID) + data, err := device.ReadFile(path) + if err != nil { + return status, fmt.Errorf("failed to read %s: %w", path, err) + } + if err := json.Unmarshal(data, &status); err != nil { + return status, fmt.Errorf("failed to parse %s: %w", path, err) + } + return status, nil +} + +// placementQualityName names a wire-level placement quality value for assertion +// messages, mirroring pillar's CPUPlacementQuality.String(). +func placementQualityName(quality int) string { + switch quality { + case placementQualityUnspecified: + return "unspecified" + case placementQualityOptimal: + return "optimal" + case placementQualityNeedsRepack: + return "needs-repack" + } + return fmt.Sprintf("unknown(%d)", quality) +} + +func appUUIDs(apps []*placedApp) []uuid.UUID { + out := make([]uuid.UUID, 0, len(apps)) + for _, app := range apps { + out = append(out, app.uuid) + } + return out +} + +// dedicatedOf returns the dedicated CPUs of the named app, for use in assertion +// messages about a conflict between two of them. +func dedicatedOf(apps []*placedApp, appName string) []uint32 { + for _, app := range apps { + if app.spec.appName == appName { + return app.dedicated + } + } + return nil +} + +func uniqueCPUs(cpus []uint32) []uint32 { + seen := map[uint32]bool{} + var out []uint32 + for _, cpu := range cpus { + if !seen[cpu] { + seen[cpu] = true + out = append(out, cpu) + } + } + return out +} + +// subtractCPUs returns the CPUs in from that are not in remove. +func subtractCPUs(from, remove []uint32) []uint32 { + excluded := map[uint32]bool{} + for _, cpu := range remove { + excluded[cpu] = true + } + var out []uint32 + for _, cpu := range from { + if !excluded[cpu] { + out = append(out, cpu) + } + } + return out +} + +func sortedKeys(cpus map[uint32]string) []uint32 { + out := make([]uint32, 0, len(cpus)) + for cpu := range cpus { + out = append(out, cpu) + } + for i := range out { + for j := i + 1; j < len(out); j++ { + if out[j] < out[i] { + out[i], out[j] = out[j], out[i] + } + } + } + return out +} + +// intsOf converts to []interface{} so Gomega's ContainElements can take it. +func intsOf(cpus []uint32) []interface{} { + out := make([]interface{}, 0, len(cpus)) + for _, cpu := range cpus { + out = append(out, cpu) + } + return out +} diff --git a/evetest/tests/apps/helpers_test.go b/evetest/tests/apps/helpers_test.go index 9dcb8bbd87d..e4f4868493b 100644 --- a/evetest/tests/apps/helpers_test.go +++ b/evetest/tests/apps/helpers_test.go @@ -67,6 +67,15 @@ func addLocalNI(devConfig *evetest.EdgeDeviceConfig) uuid.UUID { // port forwarded from the edge node, and an allow-all ACL (needed among other // things to reach the metadata server). func singleVIFWithSSH(niUUID uuid.UUID) []evetest.AppNetworkAdapter { + return singleVIFWithSSHOnPort(niUUID, appSSHFwdPort) +} + +// singleVIFWithSSHOnPort is singleVIFWithSSH with the forwarded port spelled +// out. The forwarded port belongs to the edge node, not to the app, so a test +// deploying several reachable apps at once must give each one its own -- two +// apps forwarding the same port cannot both be reached. +func singleVIFWithSSHOnPort(niUUID uuid.UUID, + edgeNodePort uint16) []evetest.AppNetworkAdapter { return []evetest.AppNetworkAdapter{ evetest.VirtualNetworkAdapter{ LogicalLabel: "vif0", @@ -74,7 +83,7 @@ func singleVIFWithSSH(niUUID uuid.UUID) []evetest.AppNetworkAdapter { PortFwdRules: []evetest.PortFwdRule{ { Protocol: evetest.NetworkProtocolTCP, - EdgeNodePort: appSSHFwdPort, + EdgeNodePort: edgeNodePort, AppPort: 22, }, }, diff --git a/evetest/tests/apps/testsuite_test.go b/evetest/tests/apps/testsuite_test.go index 97b001800ca..38c08e6e400 100644 --- a/evetest/tests/apps/testsuite_test.go +++ b/evetest/tests/apps/testsuite_test.go @@ -82,6 +82,13 @@ import ( // - TestAppRestart -- controller-requested restarts (restart counter // bumps, no purge) bring the app back to RUNNING; regression test for // a stale QMP handler quitting the re-created domain. +// - TestCPUPlacementOnePerCore -- an app asking for dedicated whole +// physical cores gets one vCPU per distinct core, each pinned 1:1. +// - TestCPUPlacementWholeCoreSMT -- the same, but both SMT siblings of each +// core become vCPUs and the guest is told which vCPUs are siblings. +// - TestCPUPlacementMultiApp -- whole-core-SMT, one-per-core and best-effort +// apps deployed together: each placed as its policy asks, on disjoint CPUs +// and disjoint physical cores, with housekeeping left intact. // - TestVMAppPurgeReplacesVMIRS -- a plain purge of a healthy app leaves // exactly one VMIRS, named for the new generation. Kubevirt only; skips // on any other hypervisor. @@ -125,6 +132,17 @@ func TestAppsSuite(test *testing.T) { evetest.TestCase{ Test: TestAppRestart, }, + evetest.TestCase{ + Test: TestCPUPlacementOnePerCore, + }, + evetest.TestCase{ + Test: TestCPUPlacementWholeCoreSMT, + }, + // Right after TestCPUPlacementWholeCoreSMT: it needs the same device + // (8 CPUs, 2 threads per core), so the framework can reuse the VM. + evetest.TestCase{ + Test: TestCPUPlacementMultiApp, + }, evetest.TestCase{ Test: TestVMAppPurgeReplacesVMIRS, }, From b5cab158baef381eee2b1b5afc9b7a1351b49343 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:06:22 +0000 Subject: [PATCH 11/15] evetest: e2e test for CPU placement stability and order independence The property an operator relies on is that a validated placement stays put. This test asserts it three ways on one unchanged set of applications: across a reboot, across a staggered start where one app is deliberately delayed, and across a restart in the reverse order. Every vCPU must land on the same host CPU each time. Order independence is what makes reboot stability real rather than incidental. Boot orders vary with image download times, network readiness and configured start delays, so a placement derived from arrival order would be reproducible only by luck. The delayed-start case exercises exactly the window in which a workload's config is known but its domain does not exist yet. Signed-off-by: Mikhail Malyshev --- .../tests/apps/cpuplacementstability_test.go | 760 ++++++++++++++++++ evetest/tests/apps/testsuite_test.go | 7 + 2 files changed, 767 insertions(+) create mode 100644 evetest/tests/apps/cpuplacementstability_test.go diff --git a/evetest/tests/apps/cpuplacementstability_test.go b/evetest/tests/apps/cpuplacementstability_test.go new file mode 100644 index 00000000000..4aae93ca103 --- /dev/null +++ b/evetest/tests/apps/cpuplacementstability_test.go @@ -0,0 +1,760 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test that a CPU placement, once decided, is a property of the set of +// applications currently claiming CPUs rather than of the history that produced +// it -- and that an application which stops really does give its CPUs back. + +package apps_test + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +const ( + // stabilityStartDelay is how long the delayed application is held back + // after the device boots. It must comfortably exceed the time EVE needs to + // bring the other applications up -- their images are already on the device + // by then, so that is well under a minute -- because the phase only proves + // anything if the others really are running while this one is still + // waiting. It is also time the test spends idle, so it is no longer than + // that argument requires. + stabilityStartDelay = 3 * time.Minute + // stabilityStartDelayTimeout bounds waiting for the delayed application to + // be released; it has to cover the delay itself plus the start. + stabilityStartDelayTimeout = stabilityStartDelay + placementRunningTimeout + // stabilityDelayReportTimeout bounds waiting for the device to report the + // delayed application as held back after a reboot. + stabilityDelayReportTimeout = 5 * time.Minute + // stabilityAppRestartTimeout bounds one deactivate/activate round trip. + stabilityAppRestartTimeout = 5 * time.Minute +) + +// TestCPUPlacementStability verifies that the host CPUs a pinned application +// runs on are decided by *which applications are running or waiting to run*, +// and not by the history of how they got there -- neither by a reboot, nor by +// the order in which they start, nor by one of them being restarted. +// +// This is the property the batch planner exists to provide (see +// cpuallocator.Plan and docs/cpu-affinity-design.md §7). The plan is not +// persisted: it is recomputed from the app configs and the topology on every +// pinned activation, over the set of *activated* applications, and a workload +// claims the slot the plan set aside for it when it starts. Three things +// follow, and none of them is visible to a test that deploys a set once and +// looks at it once: +// +// - The same configured set must yield the same assignment on every boot, or +// an application's cores silently move under it across a reboot -- exactly +// the thing a workload that was tuned (IRQ affinity, guest-side pinning, +// NUMA-local buffers) for those cores cannot tolerate. +// - An application that starts late must find its cores waiting for it. Under +// greedy per-activation allocation whoever started first won the scarce +// cores, so a delayed application could arrive to find the only core it can +// use already taken. That race is the reason planning is done over the whole +// set at once. +// - Restarting one application must not disturb the others, and must give it +// its own CPUs back: everything else keeps holding what it holds, so the +// plan the restarted workload arrives into is the same one it left. +// +// The converse is equally deliberate, and phase 4 below exists to keep it from +// being "fixed" into a stability claim: a *stopped* application releases its +// CPUs, exactly as it releases an assigned PCI device. It is not in the demand +// set while it is stopped, so it holds nothing, and the cores it used to own +// are genuinely available to anything else the controller deploys. A node with +// the capacity for a workload must never refuse it because of an application +// that is not running. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- placement is not network dependent; a +// single mgmt+apps port is enough to run the apps and reach them over SSH. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps), local NI "local-ni". +// - timer.deviceinfo.interval lowered to its minimum, so the node's CPU pool +// report (ZInfoDevice.cpu_pools) follows a workload stopping promptly. +// - 8 CPUs as 4 dual-thread cores. EVE reserves the lowest core, leaving +// three allocatable ones: one for the whole-core-SMT app, two for the +// one-per-core app. The set therefore fills the machine exactly, which is +// what makes the ordering questions meaningful -- there is no spare core to +// absorb a workload that took somebody else's. +// - A best-effort app shares the reserved core with EVE, and is there to +// confirm the housekeeping side of each phase still holds. +// +// Phases / assertions +// ------------------- +// 1. baseline: all three apps deployed together and verified with the same +// per-app and set-wide checks the other placement tests use. +// 2. reboot: the device is rebooted with the configuration unchanged. Every +// pinned app must come back on exactly the same host CPUs, with per-vCPU +// 1:1 affinity re-established -- not merely a matching cpuset, which would +// leave the vCPUs free to float within it. +// 3. restart under load: one pinned app -- the whole-core-SMT one -- is +// stopped while the other keeps running. While it is down its cores must be +// *released*: gone from the node's dedicated pool, back in the free +// housekeeping pool, one more whole core reported free, and claimed by +// nobody else, while the app that kept running keeps exactly the CPUs it +// had. Started again, it must come back on exactly the same host CPUs, +// because the running app still pins the rest of the layout. +// 4. reverse restart: both pinned apps are stopped and started again in +// the opposite order. The resulting placement must be *valid*; it is +// deliberately not required to equal the baseline, and this phase is left +// out of the cross-phase comparison. See the phase itself for why. +// 5. delayed start: the whole-core-SMT app is given a start delay and the +// device is rebooted. The other apps demonstrably start first (the delayed +// one is reported START_DELAYED while they are RUNNING), and the delayed app +// must still get its own CPUs back. It is the whole-core-SMT app that is +// delayed on purpose: it is the most constrained workload, so it is the one +// a greedy allocator would strand. It runs last because it is the only phase +// that leans on a second EVE mechanism -- the start delay -- and a device +// that fails to honor that would otherwise take the earlier verdicts with it. +// +// Every phase re-runs the full structural verification, so a phase can also +// catch a placement that is stable but wrong (e.g. the same CPUs, no longer +// pinned 1:1 after a reboot). +// +// Every phase that observes the set with *everything still holding its CPUs* is +// compared against every such phase before it -- they all observe the same +// running set, so they must all agree, and which of them is "right" is not the +// question. Phase 4, the reverse restart, is the one exception and is excluded +// from the comparison entirely, for the reason given there. A disagreement fails +// the test without stopping it: each phase costs a device boot, so a run that +// stopped at the first one would report a single broken property and hide the +// others. +// +// Test params +// ----------- +// - HYPERVISOR. Skipped under Kubevirt, where concrete CPU selection belongs +// to the kubelet rather than to the pillar allocator this test exercises. +// +// Suite placement +// --------------- +// - TestAppsSuite, right after the other CPU placement tests: it wants the +// same device (8 CPUs, 2 threads per core), so the VM can be reused. +func TestCPUPlacementStability(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + if hypervisor == evetest.HypervisorKubevirt { + evetestT.Skip("under Kubevirt the kubelet selects CPUs, not the pillar allocator") + } + + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + MinCPUs: 8, + ThreadsPerCore: 2, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // The node's CPU pool report rides on ZInfoDevice, and a change to it is not + // itself a publish trigger, so without this the release assertions of phase + // 3 would be waiting on the 10 minute default publish interval. + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.DevInfoInterval, placementDevInfoInterval) + devConfig.SetConfigProperties(cfgProps) + + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := addLocalNI(devConfig) + + // Deployed in this order, so that phase 4's reverse order really is the + // adversarial one: the flexible one-per-core app activating before the + // whole-core-SMT app that can only use a genuine two-thread core. + smtSpec := wholeCoreSMTApp("cpu-stable-smt-app", 2, appSSHFwdPort) + coreSpec := onePerCoreApp("cpu-stable-core-app", 2, appSSHFwdPort+1) + sharedSpec := sharedApp("cpu-stable-shared-app", 1, appSSHFwdPort+2) + + deployed := make([]*placedApp, 0, 3) + for _, spec := range []cpuPlacementApp{smtSpec, coreSpec, sharedSpec} { + appUUID := devConfig.AddApplication(placementAppConfig(spec, niUUID, 0)) + deployed = append(deployed, &placedApp{spec: spec, uuid: appUUID}) + } + smtApp, coreApp, sharedInst := deployed[0], deployed[1], deployed[2] + device.ApplyConfig(devConfig, true, true) + evetest.Checkpoint("config-applied") + + // Phase 1: the first observation of the set; every later phase must agree + // with it, and with each other. + waitUntilAppsRunning(device, deployed) + verifyPlacement(t, device, deployed, "baseline", true) + violations := placementViolations{t: evetestT} + violations.compare("baseline", deployed) + evetest.Checkpoint("baseline-recorded") + + // Phase 2: a reboot changes nothing about the configured set, so it must + // change nothing about the placement either. + evetest.Logger().Infof("Rebooting the device with the placement unchanged") + device.SoftReboot(true) + waitUntilPlacementRestored(t, device, deployed) + verifyPlacement(t, device, deployed, "after-reboot", true) + violations.compare("after-reboot", deployed) + evetest.Checkpoint("reboot-placement-verified") + + // Phase 3: restart one pinned app while the other keeps running. + // + // This is the strong form of the stability property, and the one EVE really + // owes a workload: the one-per-core app never stops, so it goes on holding + // its cores; the set of activated apps is otherwise unchanged; and the plan + // is a function of that set. The restarted app must therefore land back on + // exactly the CPUs it left -- same host CPUs, same vCPU order, same parked + // siblings. + // + // The whole-core-SMT app is the one restarted, for two reasons: it is the + // most constrained workload (it needs a core with two real hardware + // threads), so it is the one a greedy allocator would fail to hand its cores + // back to; and stopping it releases a whole physical core, which is what + // makes the release assertions below crisp. + // + // The best-effort app keeps running throughout -- it holds no dedicated + // CPUs, and leaving it up keeps the housekeeping side of the invariants + // under test. + restarted, kept := smtApp, coreApp + released := append([]uint32(nil), restarted.dedicated...) + keptDedicated := append([]uint32(nil), kept.dedicated...) + keptOrdered := append([]uint32(nil), kept.status.OrderedCPUs...) + + // What the node says while both are up. This also cross-checks the node-wide + // report against domainmgr's per-app status -- a report that did not account + // for the running workloads would make the release assertions meaningless. + poolsWhileRunning := awaitCPUPoolReport(t, device, + "both pinned applications holding their CPUs", + func(g Gomega, report cpuPoolReport) { + g.Expect(report.dedicated.GetCpuIds()).To(ContainElements(intsOf(released)), + "the node's dedicated pool %v omits CPUs %v, which %q holds", + report.dedicated.GetCpuIds(), released, restarted.spec.appName) + g.Expect(report.dedicated.GetCpuIds()).To(ContainElements(intsOf(keptDedicated)), + "the node's dedicated pool %v omits CPUs %v, which %q holds", + report.dedicated.GetCpuIds(), keptDedicated, kept.spec.appName) + }) + + evetest.Logger().Infof("Stopping %q while %q keeps running", + restarted.spec.appName, kept.spec.appName) + device.DeactivateApplication(restarted.uuid, true, stabilityAppRestartTimeout) + + // A stopped application releases its CPUs, exactly as it releases an + // assigned PCI device: it is no longer in the demand set, so it holds + // nothing. Asserted explicitly, because the whole point of releasing is that + // a node with the capacity for a workload must not refuse it on account of + // an application that is not running. + t.Eventually(func(g Gomega) { + status, err := readDomainCPUStatus(device, restarted.uuid) + if err != nil { + // The DomainStatus went away with the domain, which is the clearest + // possible statement that nothing is claimed. + return + } + g.Expect(status.CPUs).To(BeEmpty(), + "%q is stopped but domainmgr still records host CPUs %v for it", + restarted.spec.appName, status.CPUs) + }, stabilityAppRestartTimeout, placementPolling).Should(Succeed()) + + // Nobody else may have picked the freed CPUs up either: a running workload + // is never moved, so the app that kept running must hold exactly what it + // held before, and in the same vCPU order. + keptNow, err := readDomainCPUStatus(device, kept.uuid) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(keptNow.OrderedCPUs).To(Equal(keptOrdered), + "%q kept running while %q was stopped, so its own placement must not have "+ + "changed: it ran on %v and now runs on %v", + kept.spec.appName, restarted.spec.appName, keptOrdered, keptNow.OrderedCPUs) + keptNowDedicated := subtractCPUs(keptNow.CPUs, keptNow.EmulatorCPUs) + t.Expect(intersectCPUs(released, keptNowDedicated)).To(BeEmpty(), + "%q took over host CPUs %v released by the stopped %q; released CPUs "+ + "become free capacity, they are not handed to a running workload", + kept.spec.appName, intersectCPUs(released, keptNowDedicated), + restarted.spec.appName) + + // And the node must say so to the controller: this is the report a + // controller reads to decide whether another workload fits here. + poolsWhileStopped := awaitCPUPoolReport(t, device, + "the stopped application's CPUs back as free capacity", + func(g Gomega, report cpuPoolReport) { + g.Expect(intersectCPUs(released, report.dedicated.GetCpuIds())).To(BeEmpty(), + "host CPUs %v are still reported as dedicated although %q, which held "+ + "them, is stopped (dedicated pool: %v)", + intersectCPUs(released, report.dedicated.GetCpuIds()), + restarted.spec.appName, report.dedicated.GetCpuIds()) + g.Expect(report.housekeeping.GetFreeCpuIds()).To(ContainElements(intsOf(released)), + "host CPUs %v released by the stopped %q are not reported as free "+ + "(free housekeeping CPUs: %v)", released, restarted.spec.appName, + report.housekeeping.GetFreeCpuIds()) + g.Expect(report.dedicated.GetCpuIds()).To(ContainElements(intsOf(keptDedicated)), + "the node stopped reporting CPUs %v as dedicated although %q is still "+ + "running on them (dedicated pool: %v)", keptDedicated, + kept.spec.appName, report.dedicated.GetCpuIds()) + }) + // Free threads alone would not prove the capacity is usable: what the + // stopped workload gave back is a whole physical core, and only the + // whole-core count says another whole-core workload could now be deployed + // here -- which is the property an operator actually cares about. + t.Expect(poolsWhileStopped.housekeeping.GetFreeWholeCores()).To(BeNumerically(">", + poolsWhileRunning.housekeeping.GetFreeWholeCores()), + "%q gave back the whole physical core behind host CPUs %v, so the node must "+ + "report more free whole cores than the %d it reported while the app was "+ + "running, but it reports %d", restarted.spec.appName, released, + poolsWhileRunning.housekeeping.GetFreeWholeCores(), + poolsWhileStopped.housekeeping.GetFreeWholeCores()) + evetest.Checkpoint("stopped-app-released-cpus") + + evetest.Logger().Infof("Starting %q again", restarted.spec.appName) + device.ActivateApplication(restarted.uuid, true, stabilityAppRestartTimeout) + verifyPlacement(t, device, deployed, "after-single-app-restart", true) + violations.compare("after-single-app-restart", deployed) + // The report has to follow a workload taking CPUs just as it followed one + // giving them up; a report that only ever grows would pass everything above. + awaitCPUPoolReport(t, device, "the restarted application's CPUs claimed again", + func(g Gomega, report cpuPoolReport) { + g.Expect(report.dedicated.GetCpuIds()).To(ContainElements(intsOf(released)), + "%q is running on host CPUs %v again, but the node's dedicated pool "+ + "is %v", restarted.spec.appName, released, + report.dedicated.GetCpuIds()) + }) + evetest.Checkpoint("single-app-restart-placement-verified") + + // Phase 4: stop *both* pinned apps and start them again in the opposite + // order, with the flexible one-per-core app going first. + // + // The result must be a valid placement -- every per-app and set-wide + // invariant still holds -- but it is deliberately NOT required to equal the + // baseline, and the phase is deliberately left out of the cross-phase + // comparison. Please do not "fix" that back. + // + // The reason is the release semantics phase 3 just asserted. With both + // pinned apps stopped their cores are genuinely unowned, and the demand set + // contains only activated apps, so the first one to come back is planned as + // if it were alone on the node -- because it is. It may claim a slot the + // full plan would have set aside for the other, and the one starting second + // can then only take what is still free. That is a correct outcome, not a + // stability violation: the alternative would be for a stopped application to + // keep reserving CPUs nobody is using, which is precisely what EVE refuses + // to do. Placement quality is likewise not required to be optimal here -- + // "needs-repack" is the node correctly reporting this situation. + evetest.Logger().Infof("Restarting the pinned applications in reverse order") + for _, app := range []*placedApp{smtApp, coreApp} { + device.DeactivateApplication(app.uuid, true, stabilityAppRestartTimeout) + } + for _, app := range []*placedApp{coreApp, smtApp} { + device.ActivateApplication(app.uuid, true, stabilityAppRestartTimeout) + } + verifyPlacement(t, device, deployed, "after-reverse-restart", false) + evetest.Checkpoint("reverse-order-placement-verified") + + // Phase 5: the same set, but one pinned app is held back so the others are + // already running when it starts. Its cores must have been kept for it. + // + // Last, because it is the phase that depends on a second EVE mechanism (the + // start delay) rather than on placement alone: if the device fails to honor + // the delay, this phase cannot reach its placement assertion at all, and the + // verdicts of the earlier phases would be lost with it. + evetest.Logger().Infof("Giving %q a %s start delay and rebooting", + smtApp.spec.appName, stabilityStartDelay) + devConfig.UpdateApplication(smtApp.uuid, placementAppConfig(smtSpec, niUUID, + uint32(stabilityStartDelay.Seconds()))) + device.ApplyConfig(devConfig, true, true) + + // Subscribed before the reboot so the transition into START_DELAYED cannot + // be missed while the device is away. + delayedUpdates, stopDelayedWatch := device.WatchAppInfo(smtApp.uuid) + defer stopDelayedWatch() + device.SoftReboot(true) + + // Check what the device decided the start moment is before waiting on the + // state it should produce. It separates the two ways this phase can fail: a + // delay that was honored but turned out too short to stagger anything, and a + // delay the device dropped -- the latter shows up as a start moment derived + // from the zero time, which no clock can produce. + var startTime time.Time + t.Eventually(func(g Gomega) { + var err error + startTime, err = appStartTime(device, smtApp.uuid) + g.Expect(err).ToNot(HaveOccurred()) + }, stabilityDelayReportTimeout, placementPolling).Should(Succeed()) + evetest.Logger().Infof("the device will start %q at %s", smtApp.spec.appName, startTime) + t.Expect(startTime).To(BeTemporally(">", time.Now()), + "the device recorded %s as the moment %q may start; a %s delay configured "+ + "for a fresh boot must land in the future, and a start moment at (or "+ + "near) the zero time means the configured delay was dropped rather "+ + "than applied", startTime, smtApp.spec.appName, stabilityStartDelay) + + // The device must also report the app as held back. The app was RUNNING when + // the reboot was issued, so a START_DELAYED report can only be post-reboot. + t.Eventually(delayedUpdates, stabilityDelayReportTimeout).Should( + Receive(matchers.SatisfyPredicate( + "the start-delayed application is reported as held back", + func(info *eveinfo.ZInfoApp) bool { + return info.GetState() == eveinfo.ZSwState_START_DELAYED + })), + "%q was configured with a %s start delay but the device never reported "+ + "it as delayed after the reboot", smtApp.spec.appName, stabilityStartDelay) + + waitUntilPlacementRestored(t, device, []*placedApp{coreApp, sharedInst}) + // The evidence that this phase tests what it claims to: the others are up + // and the delayed one has not started, so whatever it gets next, it gets + // after them. + t.Expect(device.GetAppInfo(smtApp.uuid).GetState()). + To(Equal(eveinfo.ZSwState_START_DELAYED), + "%q must still be held back while the other applications are running, "+ + "otherwise this phase does not exercise a staggered start at all", + smtApp.spec.appName) + logPlacementDiagnostics(device, appUUIDs(deployed), placementShellTimeout) + evetest.Checkpoint("delayed-app-held-back") + + device.WaitUntilAppIsRunning(smtApp.uuid, stabilityStartDelayTimeout) + verifyPlacement(t, device, deployed, "after-delayed-start", true) + violations.compare("after-delayed-start", deployed) + evetest.Checkpoint("delayed-start-placement-verified") + + for _, app := range deployed { + deleteAppAndWait(t, device, devConfig, app.uuid) + } +} + +// placementAppConfig builds the deployable configuration for one placement app +// spec. The whole configuration has to be reproducible from the spec because +// UpdateApplication takes a complete ApplicationInstanceConfig and refuses any +// change to the fixed resources -- so changing only the start delay means +// rebuilding everything else identically. +func placementAppConfig(spec cpuPlacementApp, niUUID uuid.UUID, + startDelaySeconds uint32) evetest.ApplicationInstanceConfig { + return evetest.ApplicationInstanceConfig{ + DisplayName: spec.appName, + Activate: true, + Image: evetest.DockerContainer{ImageName: ubuntuCtrImage, Tag: ubuntuCtrTag}, + VirtualizationMode: eveconfig.VmMode_HVM, + CPUs: uint(spec.vCPUs), + MemoryBytes: 512 * evetest.MiB, + NetworkAdapters: singleVIFWithSSHOnPort(niUUID, spec.sshFwdPort), + CPUPlacement: spec.placement, + StartDelayInSeconds: startDelaySeconds, + } +} + +// waitUntilAppsRunning blocks until every deployed application is running. +func waitUntilAppsRunning(device *evetest.EdgeDevice, deployed []*placedApp) { + for _, app := range deployed { + device.WaitUntilAppIsRunning(app.uuid, placementRunningTimeout) + } +} + +// waitUntilPlacementRestored blocks until the device itself shows every +// application placed again, and only then waits for them to be reported as +// running. +// +// The order matters after a reboot, and only after a reboot. The controller's +// last word on an application is from before the reboot -- RUNNING -- and +// WaitUntilAppIsRunning is satisfied by an application whose latest known state +// is RUNNING, so on its own it would return while the device is still bringing +// the workloads back, and the test would then read the placement of domains +// that do not exist yet. /run/domainmgr is a tmpfs the reboot wipes, so a +// DomainStatus that carries a CPU allocation again can only have been written +// by the current boot. +func waitUntilPlacementRestored(t *GomegaWithT, device *evetest.EdgeDevice, + deployed []*placedApp) { + for _, app := range deployed { + appName := app.spec.appName + t.Eventually(func(g Gomega) { + status, err := readDomainCPUStatus(device, app.uuid) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(status.CPUs).ToNot(BeEmpty(), + "the device has not placed %q on any CPU yet", appName) + }, placementRunningTimeout, placementPolling).Should(Succeed()) + } + waitUntilAppsRunning(device, deployed) +} + +// verifyPlacement re-runs the full placement verification -- per application and +// then across the set -- and leaves what it read on each placedApp. +// +// It is run again after every phase rather than only comparing CPU numbers, +// because "the same CPUs as before" is not the whole property: a reboot that +// restored the cpuset but not the per-vCPU pinning, or that dropped the guest +// SMT topology, would leave the numbers identical and the placement broken. +// +// requireOptimal says whether the workloads must additionally sit on the slots +// the batch plan set aside for them. It is false only for phase 4, which starts +// pinned workloads into a node where the others are stopped and where landing +// off-plan is therefore the correct outcome rather than a defect. +func verifyPlacement(t *GomegaWithT, device *evetest.EdgeDevice, + deployed []*placedApp, phase string, requireOptimal bool) { + evetest.Logger().Infof("Verifying CPU placement (%s)", phase) + logPlacementDiagnostics(device, appUUIDs(deployed), placementShellTimeout) + topo, err := device.HostCPUTopology() + t.Expect(err).ToNot(HaveOccurred()) + for _, app := range deployed { + assertAppPlacement(t, device, topo, app, requireOptimal) + } + assertPlacementSetInvariants(t, device, topo, deployed) +} + +// cpuPoolReport is the node's own account of how its logical CPUs are +// partitioned, as the controller receives it in ZInfoDevice.cpu_pools. +// +// The release assertions are made against this rather than only against +// domainmgr's per-app status because this is what a controller reads to answer +// "will another workload fit on that node?". A CPU that a stopped application +// no longer holds, but that the node still advertises as taken, is a CPU nobody +// can use. +// +// The isolated pool is deliberately ignored here: it is a kernel fact that cuts +// across the other two rather than partitioning with them. +type cpuPoolReport struct { + housekeeping *eveinfo.CPUPoolUtilization + dedicated *eveinfo.CPUPoolUtilization +} + +// readCPUPoolReport picks the housekeeping and dedicated pools out of the latest +// device info. A device that reports neither has not published a CPU pool report +// at all, which is itself a failure of the phases below. +func readCPUPoolReport(device *evetest.EdgeDevice) (cpuPoolReport, error) { + var report cpuPoolReport + devInfo := device.GetDeviceInfo() + if devInfo == nil { + return report, fmt.Errorf("no device info received from the device yet") + } + for _, pool := range devInfo.GetCpuPools() { + switch pool.GetKind() { + case eveinfo.CPUPoolKind_CPU_POOL_KIND_HOUSEKEEPING: + report.housekeeping = pool + case eveinfo.CPUPoolKind_CPU_POOL_KIND_DEDICATED: + report.dedicated = pool + } + } + if report.housekeeping == nil || report.dedicated == nil { + return report, fmt.Errorf( + "the device info carries no housekeeping and dedicated CPU pool (pools: %v)", + devInfo.GetCpuPools()) + } + return report, nil +} + +// awaitCPUPoolReport polls the device info until the node's CPU pool report +// satisfies check, and returns the report that did. +// +// Polling rather than reading once: the report reaches the controller on the +// periodic ZInfoDevice publish, so the last message received still describes the +// node as it was before the workload the caller just stopped or started. Every +// property checked here converges, so a stale message only delays success -- it +// cannot satisfy an assertion the current state would fail. +func awaitCPUPoolReport(t *GomegaWithT, device *evetest.EdgeDevice, what string, + check func(g Gomega, report cpuPoolReport)) cpuPoolReport { + var report cpuPoolReport + t.Eventually(func(g Gomega) { + var err error + report, err = readCPUPoolReport(device) + g.Expect(err).ToNot(HaveOccurred()) + check(g, report) + }, placementPoolReportTimeout, placementPolling).Should(Succeed(), + "the node never reported %s", what) + evetest.Logger().Infof("node CPU pool report (%s): dedicated %v; housekeeping %v "+ + "of which free %v, free whole cores %d", what, + report.dedicated.GetCpuIds(), report.housekeeping.GetCpuIds(), + report.housekeeping.GetFreeCpuIds(), report.housekeeping.GetFreeWholeCores()) + return report +} + +// intersectCPUs returns the CPUs of a that also appear in b, so that "these must +// not be claimed any more" can be reported as the offending CPUs rather than as +// a bare boolean. +func intersectCPUs(a, b []uint32) []uint32 { + inB := make(map[uint32]bool, len(b)) + for _, cpu := range b { + inB[cpu] = true + } + var out []uint32 + for _, cpu := range a { + if inB[cpu] { + out = append(out, cpu) + } + } + return out +} + +// appStartTime reads the moment the device decided an application may start. +// +// It is read for diagnosis only, never as the property under test: the point of +// the phase is what the placement does, and this only tells the reader whether +// the device set the phase up as the test asked it to. +func appStartTime(device *evetest.EdgeDevice, appUUID uuid.UUID) (time.Time, error) { + path := fmt.Sprintf("/run/zedmanager/AppInstanceStatus/%s.json", appUUID) + data, err := device.ReadFile(path) + if err != nil { + return time.Time{}, fmt.Errorf("failed to read %s: %w", path, err) + } + var status struct { + StartTime time.Time + } + if err := json.Unmarshal(data, &status); err != nil { + return time.Time{}, fmt.Errorf("failed to parse %s: %w", path, err) + } + return status.StartTime, nil +} + +// cpuAssignment is what a pinned application holds: the host CPU behind each +// vCPU, in vCPU order, and the whole set of CPUs it occupies exclusively +// (including the SMT siblings it parks). +type cpuAssignment struct { + ordered []uint32 + dedicated []uint32 +} + +// snapshotAssignments records the pinned applications' assignments so a later +// phase can be compared against them. The slices are copied: verifyPlacement +// overwrites the placedApp fields on every phase. +func snapshotAssignments(deployed []*placedApp) map[string]cpuAssignment { + out := map[string]cpuAssignment{} + for _, app := range deployed { + if !app.spec.wantPinned { + continue + } + out[app.spec.appName] = cpuAssignment{ + ordered: append([]uint32(nil), app.status.OrderedCPUs...), + dedicated: append([]uint32(nil), app.dedicated...), + } + } + return out +} + +// placementObservation is one phase's view of what the pinned applications hold. +type placementObservation struct { + phase string + assignments map[string]cpuAssignment +} + +// placementViolations checks each phase against every phase before it and +// reports disagreements without stopping the test. +// +// Every phase passed to compare observes the same set of *activated* +// applications, so all such observations must agree; which of them is "right" is +// not the question, and comparing everything to a single reference would miss a +// phase that happens to agree with the reference while disagreeing with the +// phases in between. Reporting rather than failing keeps the run going: each +// phase costs a device boot, so stopping at the first disagreement would surface +// one broken property and hide the rest. +// +// A phase that stops every pinned application at once is deliberately not passed +// here at all: it observes a different activated set on the way, so its outcome +// is checked for validity instead (see phase 4 of TestCPUPlacementStability). +type placementViolations struct { + t *evetest.T + observed []placementObservation +} + +// compare reports how this phase's placement disagrees with the earlier ones and +// then records it for the phases still to come. +func (v *placementViolations) compare(phase string, deployed []*placedApp) { + current := snapshotAssignments(deployed) + clean := true + for appName, now := range current { + // Only the earliest phase that disagrees is reported. Later ones would + // add no information: any two differing observations already prove the + // assignment is not a function of the configured set. + for _, earlier := range v.observed { + was, known := earlier.assignments[appName] + if !known { + continue + } + // The order matters, not just the set: vCPU i runs on + // OrderedCPUs[i], so the same CPUs permuted is still a different + // placement as far as the guest is concerned. + orderedDiffers := !equalCPUSlices(was.ordered, now.ordered) + // Parked siblings are part of what the workload holds -- they are + // held back so nothing else runs on its cores -- so a change here + // means the cores it owns changed even if its vCPUs did not move. + dedicatedDiffers := !equalCPUSets(was.dedicated, now.dedicated) + if orderedDiffers { + v.t.Errorf("CPU placement is not stable: %q ran on host CPUs %v "+ + "in phase %q but on %v in phase %q; the configured set is the "+ + "same in both, so the plan must produce the same assignment", + appName, was.ordered, earlier.phase, now.ordered, phase) + } + if dedicatedDiffers { + v.t.Errorf("CPU placement is not stable: %q occupied host CPUs %v "+ + "exclusively in phase %q but occupies %v in phase %q", + appName, was.dedicated, earlier.phase, now.dedicated, phase) + } + if orderedDiffers || dedicatedDiffers { + clean = false + break + } + } + } + if clean { + evetest.Logger().Infof("CPU placement (%s) agrees with every earlier phase", + phase) + } + v.observed = append(v.observed, placementObservation{ + phase: phase, + assignments: current, + }) +} + +// equalCPUSlices compares two CPU lists position by position. +func equalCPUSlices(a, b []uint32) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// equalCPUSets compares two CPU lists as sets. +func equalCPUSets(a, b []uint32) bool { + if len(a) != len(b) { + return false + } + seen := map[uint32]int{} + for _, cpu := range a { + seen[cpu]++ + } + for _, cpu := range b { + seen[cpu]-- + if seen[cpu] < 0 { + return false + } + } + return true +} diff --git a/evetest/tests/apps/testsuite_test.go b/evetest/tests/apps/testsuite_test.go index 38c08e6e400..5b5e2b2a1fd 100644 --- a/evetest/tests/apps/testsuite_test.go +++ b/evetest/tests/apps/testsuite_test.go @@ -89,6 +89,8 @@ import ( // - TestCPUPlacementMultiApp -- whole-core-SMT, one-per-core and best-effort // apps deployed together: each placed as its policy asks, on disjoint CPUs // and disjoint physical cores, with housekeeping left intact. +// - TestCPUPlacementStability -- the same set survives a reboot, a staggered +// (start-delayed) start and a reverse restart order on the same host CPUs. // - TestVMAppPurgeReplacesVMIRS -- a plain purge of a healthy app leaves // exactly one VMIRS, named for the new generation. Kubevirt only; skips // on any other hypervisor. @@ -143,6 +145,11 @@ func TestAppsSuite(test *testing.T) { evetest.TestCase{ Test: TestCPUPlacementMultiApp, }, + // Also needs the 8-CPU, 2-threads-per-core device, so it follows the + // tests that already require one. + evetest.TestCase{ + Test: TestCPUPlacementStability, + }, evetest.TestCase{ Test: TestVMAppPurgeReplacesVMIRS, }, From ef6993809432a3537084c2b58d1a731e5b46cfeb Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:06:22 +0000 Subject: [PATCH 12/15] evetest: e2e test that a parked SMT sibling is consumed, not spare When a workload asks for one thread per physical core, the other thread of each of its cores is deliberately left idle. That thread is consumed, not free: a best-effort workload placed on it would evict the cache lines and compete for the execution units the request exists to protect, which is the whole reason for asking for a whole core. Asserted three ways, because each alone is insufficient: no other workload gets the parked thread in its cpuset, the node does not advertise it as free capacity to a controller, and nothing is ever observed executing on it. The middle one is what stops a controller from confidently over-committing the node. Signed-off-by: Mikhail Malyshev --- evetest/tests/apps/cpuparkedsiblings_test.go | 699 +++++++++++++++++++ evetest/tests/apps/testsuite_test.go | 8 + 2 files changed, 707 insertions(+) create mode 100644 evetest/tests/apps/cpuparkedsiblings_test.go diff --git a/evetest/tests/apps/cpuparkedsiblings_test.go b/evetest/tests/apps/cpuparkedsiblings_test.go new file mode 100644 index 00000000000..d6d514c28e8 --- /dev/null +++ b/evetest/tests/apps/cpuparkedsiblings_test.go @@ -0,0 +1,699 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test that the SMT sibling a one-per-core workload leaves unused is *consumed*, +// not spare: nothing else may be scheduled on it. + +package apps_test + +import ( + "strconv" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + "github.com/lf-edge/eve-api/go/evecommon" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +const ( + // parkedBusyLoops is how many busy loops the best-effort application spins + // up, and parkedBusySeconds how long each of them lives. The load exists so + // that phase 5 means something: an idle neighbour would not be observed on a + // parked CPU no matter how the device was configured, whereas a saturated one + // is exactly the workload the scheduler would spill onto any thread it is + // allowed to touch. The loops are bounded with timeout(1) so that they cannot + // outlive the test even if it fails and the app is left behind. + parkedBusyLoops = 2 + parkedBusySeconds = 240 + // parkedSampleCount is how many times the device is walked for the last-run + // CPU of every application thread. One sample would only say the parked CPU + // happened to be unused at that instant. + parkedSampleCount = 10 + // parkedSampleTimeout bounds the whole sampling script: parkedSampleCount + // passes over /proc, each of which forks per thread on a machine that is + // deliberately under load. + parkedSampleTimeout = 6 * time.Minute + // parkedBusyLoadFloor is the CPU-equivalent of non-idle time the best-effort + // application's cpuset must accumulate during the sampling window for the + // window to count as loaded at all. parkedBusyLoops busy loops on + // parkedBusyLoops vCPUs produce close to parkedBusyLoops CPUs of load; this + // only has to distinguish "the loops ran" from "the loops never started". + parkedBusyLoadFloor = 1.0 + // parkedIdleShareCeiling is the fraction of wall time a parked CPU may spend + // off the idle task during the sampling window. + // + // It is deliberately loose rather than near zero, because absolute idleness is + // not what EVE promises and asserting it would be flaky: kernel per-CPU + // threads legitimately live there, EVE does not confine its own services to + // the housekeeping set, and the emulator/IO threads of the workload that + // *owns* the core may use it -- it is that workload's core to waste. What the + // mode promises is that no *other* workload runs there, and a workload that + // could would show up far above this ceiling: the best-effort app's own CPUs + // sit near 100% in the very same window. Measured on the reference device the + // parked CPU spends about 0.2% of the window off the idle task, so this leaves + // two orders of magnitude of headroom for a busier node. + parkedIdleShareCeiling = 0.25 +) + +// TestCPUPlacementParkedSiblings verifies that the SMT sibling threads a +// one-per-core workload does not use are *consumed by it*, not left as spare +// capacity for someone else. +// +// This is the semantics of full_pcpus_only with threads_per_core=1, and it is +// easy to mistake for waste. The workload asks for every thread of a physical +// core while using only one of them on purpose: the sibling thread shares that +// core's L1/L2 caches and its execution engine, so anything scheduled on it +// evicts the pinned vCPU's cache lines and steals issue slots and ALU time from +// it. The interference is invisible in any CPU accounting -- the pinned vCPU +// still has "its" thread -- and shows up only as jitter and lost throughput, +// which is precisely what a workload buys whole cores to avoid. Leaving the +// sibling idle is therefore the deliberate cost of the mode, not a bug to +// reclaim, and a "helpful" allocator that handed that thread to a best-effort +// app would silently destroy the guarantee it was asked for. +// +// EVE enforces this in three places (assignmentCPUs puts the parked CPUs in the +// reserved set and in the app's own cpuset; Placer.FreeCPUs excludes the whole +// dedicated union, parked included, from what a best-effort app's cpuset is +// built from; coreIsDedicated refuses a physical core with any sibling held). +// The existing multi-app test would catch a regression in any of them, but only +// as "a best-effort app's cpuset overlaps the dedicated union" -- which does not +// say that a *deliberately idle* thread leaked. This test names that. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- placement is not network dependent; a +// single mgmt+apps port is enough to run the apps and reach them over SSH. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps), local NI "local-ni". +// - timer.deviceinfo.interval lowered to its minimum, because phase 4 asserts +// on the node's CPU pool report, which only reaches the controller on the +// periodic ZInfoDevice publish. +// - 8 CPUs as 4 dual-thread cores. EVE reserves the lowest CPU, which makes its +// whole core unallocatable, leaving three allocatable cores: one for the +// one-per-core app (1 vCPU -> 1 assigned thread + 1 parked sibling), one for +// the whole-core-SMT app, one spare. That leaves four CPUs of housekeeping +// for EVE and the best-effort app -- enough that saturating the best-effort +// app cannot starve EVE's own services and trip its watchdog. +// - Three container apps (lfedge/evetest-ubuntu-ctr), each with its own +// forwarded SSH port: a best-effort one deployed first (so it starts out with +// the whole machine in its cpuset and is only off the dedicated CPUs if the +// device actively redistributes), the one-per-core app whose parked sibling is +// the subject, and a second pinned app so that "no *other* workload" has more +// than one witness. +// +// Phases / assertions +// ------------------- +// 1. baseline: all three apps run and pass the same per-app and set-wide +// placement checks the other placement tests use (verifyPlacement). +// 2. parked set identified: the one-per-core app's parked CPUs are computed as +// its dedicated set minus its per-vCPU assignment, and each one is confirmed +// to be the SMT sibling of an assigned CPU. Non-empty, or the device has no +// SMT and the test skips rather than passing vacuously. +// 3. the parked CPUs are absent from every other workload: not in the +// best-effort app's cpuset, and not in any other app's dedicated set or +// cpuset. +// 4. the node does not advertise them as capacity: each parked CPU is in the +// dedicated pool of ZInfoDevice.cpu_pools and in no free CPU list, so a +// controller cannot be told to place another workload there. +// 5. nothing else actually runs there: while the best-effort app saturates its +// own vCPUs, the last-run CPU of every thread of every other application is +// sampled repeatedly and must never be a parked CPU, and the parked CPUs must +// stay near idle while the best-effort app's CPUs are busy. +// +// Test params +// ----------- +// - HYPERVISOR. Skipped under Kubevirt, where concrete CPU selection belongs +// to the kubelet rather than to the pillar allocator this test exercises. +// +// Suite placement +// --------------- +// - TestAppsSuite, with the other CPU placement tests: it wants the same device +// (8 CPUs, 2 threads per core), so the VM can be reused. +func TestCPUPlacementParkedSiblings(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + if hypervisor == evetest.HypervisorKubevirt { + evetestT.Skip("under Kubevirt the kubelet selects CPUs, not the pillar allocator") + } + + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + MinCPUs: 8, + ThreadsPerCore: 2, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + + // Checked before anything is deployed: on a device whose cores are all + // single-threaded there is no sibling to park, so every assertion below would + // hold trivially. That is a skip, not a pass -- and finding out after three + // applications have been brought up would only waste the boot. + topo, err := device.HostCPUTopology() + t.Expect(err).ToNot(HaveOccurred()) + if !deviceHasSMT(topo) { + evetestT.Skip("the device has no SMT sibling threads at all, so a one-per-core " + + "workload parks nothing and there is no thread that could leak") + } + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // Phase 4 reads the node's CPU pool report, which rides on the periodic + // ZInfoDevice publish; at the 10 minute default the test would spend most of + // its time waiting for a message. + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.DevInfoInterval, placementDevInfoInterval) + devConfig.SetConfigProperties(cfgProps) + + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := addLocalNI(devConfig) + + // The best-effort app is deployed first on purpose: it then starts with every + // CPU in its cpuset, so it only ends up off the parked CPUs if the device + // actively narrows it when the pinned apps take their cores. It gets one vCPU + // per busy loop, so that phase 5's load really saturates it and the scheduler + // has a reason to look for another CPU to put it on. + sharedSpec := sharedApp("cpu-parked-shared-app", parkedBusyLoops, appSSHFwdPort+2) + // One vCPU, so the app occupies exactly one physical core and the parked set + // is exactly the one sibling thread it declines to use. + coreSpec := onePerCoreApp("cpu-parked-core-app", 1, appSSHFwdPort) + // A second pinned workload, so "no other workload holds a parked CPU" is + // checked against something that also has dedicated CPUs of its own, not only + // against the best-effort app. + smtSpec := wholeCoreSMTApp("cpu-parked-smt-app", 2, appSSHFwdPort+1) + + deployed := make([]*placedApp, 0, 3) + for _, spec := range []cpuPlacementApp{sharedSpec, coreSpec, smtSpec} { + appUUID := devConfig.AddApplication(placementAppConfig(spec, niUUID, 0)) + deployed = append(deployed, &placedApp{spec: spec, uuid: appUUID}) + } + sharedInst, coreApp := deployed[0], deployed[1] + device.ApplyConfig(devConfig, true, true) + evetest.Checkpoint("config-applied") + + // Phase 1: the placement itself has to be right before anything can be + // concluded about the threads it parks. + waitUntilAppsRunning(device, deployed) + verifyPlacement(t, device, deployed, "baseline", true) + evetest.Checkpoint("baseline-verified") + + // Phase 2: name the parked CPUs. Everything below is about these specific + // CPUs, so if they cannot be identified the rest of the test proves nothing. + parked := parkedCPUsOf(t, topo, coreApp) + evetest.Logger().Infof("%q runs on host CPUs %v and parks their SMT siblings %v "+ + "(dedicated set %v)", coreApp.spec.appName, coreApp.status.OrderedCPUs, parked, + coreApp.dedicated) + evetest.Checkpoint("parked-cpus-identified") + + // Phase 3: the parked CPUs must not appear in anything else's allocation. + assertParkedCPUsNotShared(t, parked, coreApp, deployed) + evetest.Checkpoint("parked-cpus-not-shared") + + // Phase 4: nor may the node advertise them as capacity. + assertParkedCPUsNotAdvertised(t, device, parked, coreApp) + evetest.Checkpoint("parked-cpus-not-advertised") + + // Phase 5: and nothing else may actually be seen running on them. + assertParkedCPUsUnused(t, device, parked, coreApp, sharedInst, deployed) + evetest.Checkpoint("parked-cpus-unused") + + for _, app := range deployed { + deleteAppAndWait(t, device, devConfig, app.uuid) + } +} + +// deviceHasSMT reports whether any physical core of the device carries more than +// one logical CPU. +func deviceHasSMT(topo evetest.HostTopology) bool { + for _, cpu := range topo.IDs() { + if len(topo.SiblingsOf(cpu)) > 1 { + return true + } + } + return false +} + +// parkedCPUsOf derives the CPUs a one-per-core workload holds without using: the +// CPUs it occupies exclusively, minus the ones actually backing a vCPU. +// +// Each of them is confirmed to be the SMT sibling of an assigned CPU. That is +// what makes them *parked* rather than merely surplus: the workload holds them +// because they share a physical core -- its caches and its execution engine -- +// with a thread it does run on. A CPU in the dedicated set that is not a sibling +// of anything assigned would be plain over-allocation, and stating the sibling +// relationship here is what keeps this test about the mode's semantics rather +// than about set arithmetic. +func parkedCPUsOf(t *GomegaWithT, topo evetest.HostTopology, + app *placedApp) []uint32 { + assigned := app.status.OrderedCPUs + parked := subtractCPUs(app.dedicated, assigned) + t.Expect(parked).ToNot(BeEmpty(), + "%q asked for whole physical cores with one thread each and was assigned host "+ + "CPUs %v out of the dedicated set %v, so it should also be holding their "+ + "idle SMT siblings; holding nothing beyond the assigned threads means the "+ + "siblings were left available to other workloads, which is exactly the "+ + "cache and execution-unit interference full_pcpus_only is bought to prevent", + app.spec.appName, assigned, app.dedicated) + for _, cpu := range parked { + sibling := false + for _, a := range assigned { + if topo.SameCore(cpu, a) { + sibling = true + break + } + } + t.Expect(sibling).To(BeTrue(), + "host CPU %d is held exclusively by %q but backs none of its vCPUs (%v) and "+ + "is not on a physical core with any of them either (siblings of %d: %v); "+ + "a parked CPU is only justified by sharing a core with a thread the "+ + "workload runs on -- anything else is capacity taken from the node for "+ + "no reason", cpu, app.spec.appName, assigned, cpu, topo.SiblingsOf(cpu)) + } + return parked +} + +// assertParkedCPUsNotShared checks that no workload other than the one parking +// them has a parked CPU in its allocation -- neither in a dedicated set nor in a +// cgroup cpuset. +// +// The cpuset is checked separately from the dedicated set because they can fail +// independently: bookkeeping that keeps the parked CPUs out of every other +// workload's *accounting* is worthless if the cgroup still lets that workload's +// threads run there, and a cpuset that happens to exclude them today is not a +// guarantee if the allocator considers them free. +func assertParkedCPUsNotShared(t *GomegaWithT, parked []uint32, owner *placedApp, + deployed []*placedApp) { + for _, app := range deployed { + if app == owner { + continue + } + t.Expect(intersectCPUs(parked, app.dedicated)).To(BeEmpty(), + "host CPUs %v are dedicated to %q although %q parks them: they are the idle "+ + "SMT siblings of the cores it runs on, held back precisely so that "+ + "nothing else touches those cores' caches or execution units", + intersectCPUs(parked, app.dedicated), app.spec.appName, owner.spec.appName) + t.Expect(intersectCPUs(parked, app.cpuset)).To(BeEmpty(), + "the cpuset %v of %q includes host CPUs %v, which %q parks as the idle SMT "+ + "siblings of its own cores; a thread of %q scheduled there would evict "+ + "that core's cache lines and steal issue slots from the pinned vCPU on "+ + "the sibling thread, which is the whole reason the sibling is held idle "+ + "instead of being handed out", app.cpuset, app.spec.appName, + intersectCPUs(parked, app.cpuset), owner.spec.appName, app.spec.appName) + } +} + +// assertParkedCPUsNotAdvertised checks the node's own report: a parked CPU must +// be accounted for as dedicated and must appear in no pool's free CPU list. +// +// This is a different failure from a wrong cpuset. The controller decides where +// to place the *next* workload from this report, so a parked CPU advertised as +// free is a promise the node cannot keep: it would either be handed out -- +// destroying the guarantee the one-per-core app paid for -- or the placement +// would be refused at the last moment, on a node the controller was told had +// room. +func assertParkedCPUsNotAdvertised(t *GomegaWithT, device *evetest.EdgeDevice, + parked []uint32, owner *placedApp) { + awaitCPUPoolReport(t, device, "the parked SMT siblings as dedicated capacity", + func(g Gomega, report cpuPoolReport) { + g.Expect(report.dedicated.GetCpuIds()).To(ContainElements(intsOf(parked)), + "the node's dedicated CPU pool %v omits host CPUs %v, which %q holds as "+ + "the parked SMT siblings of its cores; a CPU the node does not "+ + "account for as taken is a CPU it may hand to the next workload", + report.dedicated.GetCpuIds(), parked, owner.spec.appName) + g.Expect(intersectCPUs(parked, report.housekeeping.GetFreeCpuIds())). + To(BeEmpty(), + "host CPUs %v are reported as free housekeeping capacity although "+ + "%q parks them (free housekeeping CPUs: %v); they are consumed, "+ + "not spare -- the sibling of a core carrying a pinned vCPU can "+ + "only be used by trashing that vCPU's caches and stealing its "+ + "execution-unit time", + intersectCPUs(parked, report.housekeeping.GetFreeCpuIds()), + owner.spec.appName, report.housekeeping.GetFreeCpuIds()) + g.Expect(intersectCPUs(parked, report.dedicated.GetFreeCpuIds())).To(BeEmpty(), + "host CPUs %v are reported as free within the dedicated pool although "+ + "%q parks them (free dedicated CPUs: %v)", + intersectCPUs(parked, report.dedicated.GetFreeCpuIds()), + owner.spec.appName, report.dedicated.GetFreeCpuIds()) + }) +} + +// assertParkedCPUsUnused is the observational half of the test: with the +// best-effort application saturating its own vCPUs, no thread of any other +// application may be seen on a parked CPU, and the parked CPUs must stay near +// idle while the best-effort app's CPUs are busy. +// +// Load matters here. Every assertion above reads a *configuration* -- a cpuset, a +// dedicated set, a report -- and a configuration can be right while the effect it +// exists to produce is not. What this phase adds is a workload that would take +// the parked thread if the kernel let it: a saturated best-effort app is exactly +// the neighbour the scheduler spreads onto every CPU it is permitted to use, so +// its absence from the parked CPUs is evidence rather than coincidence. The +// busy-CPU measurement is reported alongside as the positive control -- without +// it, an app whose loops never started would make this phase pass for the wrong +// reason. +func assertParkedCPUsUnused(t *GomegaWithT, device *evetest.EdgeDevice, + parked []uint32, owner, busy *placedApp, deployed []*placedApp) { + startBusyLoad(t, device, busy) + + samples, cpuTimes := sampleAppThreadCPUs(t, device, appUUIDs(deployed)) + + // The positive control: the window was loaded. A sampling window in which the + // best-effort app did nothing would prove nothing about where it can run. + busyLoad := nonIdleCPUEquivalents(cpuTimes, busy.cpuset) + evetest.Logger().Infof("during the sampling window the cpuset %v of %q carried "+ + "%.2f CPUs of non-idle time", busy.cpuset, busy.spec.appName, busyLoad) + t.Expect(busyLoad).To(BeNumerically(">=", parkedBusyLoadFloor), + "the best-effort application %q was asked to saturate its %d vCPUs but its "+ + "cpuset %v only accumulated %.2f CPUs of non-idle time; without a neighbour "+ + "that actually wants CPU time, finding the parked CPUs %v unused says "+ + "nothing about whether they are reachable", + busy.spec.appName, busy.spec.vCPUs, busy.cpuset, busyLoad, parked) + + // No thread of any other application was seen running on a parked CPU. + // + // Only observations of threads that demonstrably ran are considered (see + // ranDuringInterval): a thread's last-run CPU is a leftover, and the + // best-effort app is deployed before the pinned ones -- it starts with the + // whole machine in its cpuset -- so a thread that has not been scheduled since + // the cpuset was narrowed can still name a now-parked CPU without ever having + // been able to run there again. Only a CPU a thread ran on *during* the window + // is evidence. + // + // The owner's own threads are excluded: the parked sibling is part of the core + // it bought, so its emulator/IO threads using it is the mode working as + // intended, not a leak. Kernel threads are excluded by construction -- only + // threads in an application's cgroup are sampled -- because ksoftirqd/N and + // friends are bound to their CPU and legitimately live there. + parkedSet := map[uint32]bool{} + for _, cpu := range parked { + parkedSet[cpu] = true + } + observed := 0 + for _, sample := range samples { + if !sample.ranDuringInterval { + continue + } + if sample.appUUID == owner.uuid { + continue + } + observed++ + t.Expect(parkedSet).ToNot(HaveKey(sample.cpu), + "thread %d (%q) of application %s ran on host CPU %d, which %q parks as the "+ + "idle SMT sibling of one of its cores; a foreign thread there contends "+ + "for that core's L1/L2 caches and its execution units with the pinned "+ + "vCPU on the other thread, which is the interference the workload asked "+ + "for whole cores to avoid (parked CPUs: %v)", + sample.tid, sample.comm, sample.appUUID, sample.cpu, owner.spec.appName, + parked) + } + t.Expect(observed).To(BeNumerically(">", 0), + "none of the %d application thread observations over %d passes caught a thread "+ + "of an application other than %q that had actually run since the previous "+ + "pass, so the claim that no foreign thread ran on the parked CPUs %v rests "+ + "on no observation at all", + len(samples), parkedSampleCount, owner.spec.appName, parked) + evetest.Logger().Infof("sampled %d application thread observations over %d passes; "+ + "%d of them caught a thread of another application that had just run, none of "+ + "those on the parked CPUs %v", len(samples), parkedSampleCount, observed, parked) + + // And the parked CPUs stayed near idle throughout. This is the weakest of the + // three -- see parkedIdleShareCeiling for why it cannot be tightened to zero + // -- but it is the only one that would notice work arriving on a parked CPU + // from somewhere the per-thread sampling does not look (a host-side helper + // outside any app cgroup, or a thread that came and went between passes). + for _, cpu := range parked { + share, ok := nonIdleShare(cpuTimes, cpu) + t.Expect(ok).To(BeTrue(), + "/proc/stat carries no usable counters for the parked host CPU %d", cpu) + evetest.Logger().Infof("parked host CPU %d spent %.1f%% of the sampling window "+ + "off the idle task", cpu, share*100) + t.Expect(share).To(BeNumerically("<", parkedIdleShareCeiling), + "parked host CPU %d spent %.1f%% of the sampling window off the idle task "+ + "while %q was holding it idle; over the same window the best-effort "+ + "application's cpuset %v carried %.2f CPUs of load, so this looks like "+ + "work that leaked onto a thread whose whole purpose is to stay unused so "+ + "the pinned vCPU on its sibling keeps the core's caches and execution "+ + "units to itself", cpu, share*100, owner.spec.appName, busy.cpuset, + busyLoad) + } +} + +// parkedBusyLoadScript spins up bounded busy loops inside an application. +// +// timeout(1) bounds each loop, and the loops are detached so the SSH session +// that started them can return: the sampling below has to run while they are +// still going. The bound is what keeps a failed run from leaving a device +// spinning -- nothing here outlives parkedBusySeconds even if the test aborts. +const parkedBusyLoadScript = `i=0 +while [ "$i" -lt @LOOPS@ ]; do + nohup timeout @SECONDS@ sh -c 'while : ; do : ; done' >/dev/null 2>&1 /stat. The comm field can contain spaces and parentheses, so the +// fields before them are cut off at the closing parenthesis rather than counted, +// which shifts the indices by two. comm itself is reported last so that spaces in +// it cannot shift anything. +const parkedSamplingScript = `UUIDS='@UUIDS@' + +awk '/^cpu[0-9]/ {print "STAT begin " $0}' /proc/stat + +i=0 +while [ "$i" -lt @SAMPLES@ ]; do + for p in /proc/[0-9]*; do + cg=$(cat "$p/cgroup" 2>/dev/null | tr '\n' ' ') + owner="" + for u in $UUIDS; do + case "$cg" in + *"$u"*) owner=$u; break ;; + esac + done + [ -n "$owner" ] || continue + for td in "$p"/task/[0-9]*; do + fields=$(awk '{sub(/.*\) /, ""); print $12+$13, $37}' "$td/stat" 2>/dev/null) + [ -n "$fields" ] || continue + echo "THREAD $owner ${td##*/} $fields $(cat "$td/comm" 2>/dev/null)" + done + done + i=$((i+1)) + sleep 1 +done + +awk '/^cpu[0-9]/ {print "STAT end " $0}' /proc/stat +` + +// threadSample is one observation of one application thread: the CPU it last ran +// on, and whether it demonstrably ran since the previous pass -- which is what +// makes that CPU an observation rather than a leftover. +type threadSample struct { + appUUID uuid.UUID + tid int + comm string + cpu uint32 + // ranDuringInterval is true if the thread's cumulative CPU time grew since + // the previous pass, i.e. it was scheduled in between and therefore really + // ran on cpu. The first observation of a thread is never marked: there is + // nothing to compare it against. + ranDuringInterval bool +} + +// cpuTimeWindow holds the per-CPU /proc/stat counters from the start and the end +// of the sampling window, so how each CPU spent the window can be derived. +type cpuTimeWindow struct { + begin map[uint32][]uint64 + end map[uint32][]uint64 +} + +// sampleAppThreadCPUs runs the sampling script and parses it. +func sampleAppThreadCPUs(t *GomegaWithT, device *evetest.EdgeDevice, + appUUIDs []uuid.UUID) ([]threadSample, cpuTimeWindow) { + uuids := make([]string, 0, len(appUUIDs)) + for _, appUUID := range appUUIDs { + uuids = append(uuids, appUUID.String()) + } + script := strings.ReplaceAll(parkedSamplingScript, "@UUIDS@", strings.Join(uuids, " ")) + script = strings.ReplaceAll(script, "@SAMPLES@", strconv.Itoa(parkedSampleCount)) + stdout, stderr, err := device.RunShellScript(script, parkedSampleTimeout, 0) + t.Expect(err).ToNot(HaveOccurred(), + "failed to sample where the applications' threads run (stderr: %s)", stderr) + + window := cpuTimeWindow{ + begin: map[uint32][]uint64{}, + end: map[uint32][]uint64{}, + } + // Cumulative CPU time per thread as of its previous observation, so a rise can + // be detected without keeping every sample of every thread. + lastCPUTime := map[int]uint64{} + var samples []threadSample + for _, line := range strings.Split(stdout, "\n") { + fields := strings.Fields(line) + if len(fields) < 5 { + continue + } + switch fields[0] { + case "STAT": + cpu, err := strconv.ParseUint(strings.TrimPrefix(fields[2], "cpu"), 10, 32) + if err != nil { + continue + } + var values []uint64 + for _, field := range fields[3:] { + value, err := strconv.ParseUint(field, 10, 64) + if err != nil { + break + } + values = append(values, value) + } + if fields[1] == "begin" { + window.begin[uint32(cpu)] = values + } else { + window.end[uint32(cpu)] = values + } + case "THREAD": + appUUID, err := uuid.FromString(fields[1]) + if err != nil { + continue + } + tid, err := strconv.Atoi(fields[2]) + if err != nil { + continue + } + cpuTime, err := strconv.ParseUint(fields[3], 10, 64) + if err != nil { + continue + } + cpu, err := strconv.ParseUint(fields[4], 10, 32) + if err != nil { + continue + } + previous, seen := lastCPUTime[tid] + lastCPUTime[tid] = cpuTime + samples = append(samples, threadSample{ + appUUID: appUUID, + tid: tid, + // comm is the rest of the line: it may itself contain spaces (a + // QEMU vCPU thread is "CPU 0/KVM"). + comm: strings.Join(fields[5:], " "), + cpu: uint32(cpu), + ranDuringInterval: seen && cpuTime > previous, + }) + } + } + t.Expect(samples).ToNot(BeEmpty(), + "no application thread was found on the device at all, so nothing was "+ + "sampled; the assertions about where threads did *not* run would be "+ + "vacuous (script output: %s)", stdout) + return samples, window +} + +// nonIdleShare is the fraction of the sampling window a CPU spent off the idle +// task, from its /proc/stat counters. The second result is false if the CPU was +// not reported at both ends of the window. +func nonIdleShare(window cpuTimeWindow, cpu uint32) (float64, bool) { + begin, okBegin := window.begin[cpu] + end, okEnd := window.end[cpu] + // user, nice, system, idle, iowait, ... -- iowait is counted as idle, which + // on a CPU nothing is allowed to run on is the same thing anyway. + if !okBegin || !okEnd || len(begin) < 5 || len(end) < 5 { + return 0, false + } + var total, idle float64 + for i := range end { + if i >= len(begin) { + break + } + total += float64(end[i] - begin[i]) + if i == 3 || i == 4 { + idle += float64(end[i] - begin[i]) + } + } + if total <= 0 { + return 0, false + } + return (total - idle) / total, true +} + +// nonIdleCPUEquivalents sums the non-idle share of the given CPUs, i.e. how many +// whole CPUs' worth of work happened on them during the window. It is the +// measure of the load the best-effort application generated, and has to be a sum +// rather than a per-CPU share because that load is free to float across the +// app's whole cpuset. +func nonIdleCPUEquivalents(window cpuTimeWindow, cpus []uint32) float64 { + var sum float64 + for _, cpu := range cpus { + if share, ok := nonIdleShare(window, cpu); ok { + sum += share + } + } + return sum +} diff --git a/evetest/tests/apps/testsuite_test.go b/evetest/tests/apps/testsuite_test.go index 5b5e2b2a1fd..5693edb3d23 100644 --- a/evetest/tests/apps/testsuite_test.go +++ b/evetest/tests/apps/testsuite_test.go @@ -91,6 +91,10 @@ import ( // and disjoint physical cores, with housekeeping left intact. // - TestCPUPlacementStability -- the same set survives a reboot, a staggered // (start-delayed) start and a reverse restart order on the same host CPUs. +// - TestCPUPlacementParkedSiblings -- the SMT sibling a one-per-core app leaves +// unused is consumed by it, not spare: no other workload gets it in its +// cpuset, the node does not advertise it as free, and nothing is ever +// observed running on it. // - TestVMAppPurgeReplacesVMIRS -- a plain purge of a healthy app leaves // exactly one VMIRS, named for the new generation. Kubevirt only; skips // on any other hypervisor. @@ -150,6 +154,10 @@ func TestAppsSuite(test *testing.T) { evetest.TestCase{ Test: TestCPUPlacementStability, }, + // Same device again. + evetest.TestCase{ + Test: TestCPUPlacementParkedSiblings, + }, evetest.TestCase{ Test: TestVMAppPurgeReplacesVMIRS, }, From 3ba8aac3bbeb8ebbc7c206e4e47355b12d0ad0dc Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:06:23 +0000 Subject: [PATCH 13/15] evetest: e2e test that a fragmented node reports needs_repack A node can have enough free threads for a whole-core workload and still not have a single free whole core, because earlier thread-granular workloads left one thread busy on each. The two shortages call for opposite responses: nothing will help the first, while rearranging existing workloads would resolve the second, and only the workloads' owner can decide whether that disruption is acceptable. The test fragments the node deliberately, confirms the refusal carries cpu.placement.needs_repack rather than a plain shortage, and then repacks and confirms the workload really does run -- so the advice the code gives is demonstrated to be true, not merely plausible. Signed-off-by: Mikhail Malyshev --- evetest/tests/apps/cpuneedsrepack_test.go | 740 ++++++++++++++++++++++ evetest/tests/apps/testsuite_test.go | 9 + 2 files changed, 749 insertions(+) create mode 100644 evetest/tests/apps/cpuneedsrepack_test.go diff --git a/evetest/tests/apps/cpuneedsrepack_test.go b/evetest/tests/apps/cpuneedsrepack_test.go new file mode 100644 index 00000000000..844e9b08e01 --- /dev/null +++ b/evetest/tests/apps/cpuneedsrepack_test.go @@ -0,0 +1,740 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test that a CPU shortage a repack would fix is reported as such -- and that +// the node's own capacity report explains it in whole cores, not free threads. + +package apps_test + +import ( + "fmt" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// Placement failure codes as they travel on the wire, in +// ZInfoApp.AppErr[].error_code. +// +// Spelled out here rather than imported from pillar's types package for two +// reasons: evetest builds against a released pillar module, which does not carry +// constants added on a feature branch; and these strings *are* the published +// contract a controller matches on, so a test that referenced the device's own +// constant could not notice one of them being renamed under it. +const ( + // errCodeNeedsRepack means the workload would fit if the pinned workloads + // currently running were restarted together. Actionable. + errCodeNeedsRepack = "cpu.placement.needs_repack" + // errCodeInsufficient means no arrangement on this node fits it. Not + // actionable without changing the node or the request. + errCodeInsufficient = "cpu.placement.insufficient" +) + +const ( + // repackFragmenterVCPUs is the size of each thread-granular workload used to + // fragment the node, and repackFragmenters how many of them are deployed. + // + // Together they must claim every allocatable thread but one, because the + // thread-granular allocator hands out the lowest-numbered free CPUs: a set of + // them therefore occupies a contiguous run, and only its top end can leave a + // core half-owned. On the 8-CPU device below that is 6 of the 7 allocatable + // threads (cpu0 is reserved for EVE), i.e. three 2-vCPU workloads -- after + // which stopping one of them opens the gaps this test is about (see + // chooseFragmentingVictim). + repackFragmenterVCPUs = 2 + repackFragmenters = 3 + // repackWholeCoreVCPUs is the whole-core-SMT request that must fail on the + // fragmented node and succeed after the repack. Two vCPUs = one physical + // core: the smallest whole-core request there is, so its failure cannot be + // blamed on the size of the ask. + repackWholeCoreVCPUs = 2 + // repackOversizedVCPUs is a one-per-core request for more physical cores than + // the device has at all -- the contrast case, where no repack helps. + repackOversizedVCPUs = 8 + // repackErrorTimeout bounds waiting for the device to report a placement + // failure. It covers creating the app's volume from the already-downloaded + // image, the trip through zedmanager/domainmgr and the info message back. + repackErrorTimeout = 10 * time.Minute + // repackStopTimeout bounds one deactivation. + repackStopTimeout = 5 * time.Minute +) + +// TestCPUPlacementNeedsRepack verifies the distinction between the two CPU +// placement failure codes EVE publishes, on a node deliberately fragmented so +// that the distinction is the only thing separating a right answer from a wrong +// one. +// +// The codes exist because a controller has to react differently to them: +// cpu.placement.needs_repack says a placement for this workload does exist and +// is only blocked by where the *running* workloads sit, so restarting the pinned +// workloads together would let it run; cpu.placement.insufficient says nothing +// on this node fits it and only a bigger node or a smaller request will help. +// Reporting the first as the second is not a cosmetic error: the controller's +// one remedy for a fragmented node is a repack, and it has no reason to attempt +// one on a node that says nothing fits. +// +// How a node gets fragmented is the crux, and it is not obvious. A whole-core +// workload takes and releases *whole* physical cores, so no arrangement of +// whole-core workloads can leave a core partly used. Only a thread-granular +// dedicated workload -- CPU_POLICY_DEDICATED without full_pcpus_only -- claims +// individual logical CPUs, and a set of them can leave several cores with one +// thread taken and one free. Such a core is refused to every whole-core request +// (its sibling is not the requester's to have), so the node can hold plenty of +// free threads and yet not a single free whole core. That state is exactly what +// the node's whole-core counters exist to report, and what this test is the +// first to observe end to end. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- placement is not network dependent; a +// single mgmt+apps port is enough to run the apps and reach one over SSH. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps), local NI "local-ni". +// - timer.deviceinfo.interval lowered to its minimum: the node's CPU pool +// report rides the periodic ZInfoDevice publish, and phases 2 and 3 assert +// on it right after a workload stops or fails to start. +// - 8 CPUs as 4 dual-thread cores. EVE reserves the lowest CPU, which makes +// its whole core unallocatable, leaving three allocatable cores (six +// threads) plus the reserved core's free sibling: seven threads a dedicated +// workload can be given. +// - Four container apps (lfedge/evetest-ubuntu-ctr), each with its own +// forwarded SSH port: three thread-granular ones that fragment the node and +// one whole-core-SMT one that must fail on it, plus -- in phase 4 -- an +// oversized whole-core app that no node of this size can ever host. +// +// Phases / assertions +// ------------------- +// 1. fragmenters-placed: the three thread-granular apps run, each with host +// CPUs of its own at thread granularity -- pinned, but with no per-vCPU +// assignment and no synthesized guest SMT topology, which is what makes them +// able to half-own a core (assertThreadGranularPlacement), and on disjoint +// CPU sets (assertPlacementSetInvariants). +// 2. node-fragmented: one fragmenter is stopped -- which one is derived from the +// allocation the device actually made, not assumed -- so that what remains +// half-owns physical cores. The node must then report zero free whole cores +// while still reporting more free threads than the whole-core request needs: +// the condition under which a free-thread count answers "will it fit?" +// wrongly. +// 3. needs-repack-reported: a 2-vCPU whole-core-SMT app deployed onto that node +// must fail to start and must report cpu.placement.needs_repack and not +// cpu.placement.insufficient, with its placement quality left unspecified (it +// never ran, so it has no placement to rate) and no CPUs held. +// 4. insufficient-reported: a whole-core request for more cores than the device +// has must report cpu.placement.insufficient -- the other code, from the same +// boot, so the two are known to be told apart rather than both spelled the +// same way. +// 5. repack-honoured: the pinned workloads are stopped and started again +// together, which is the repack the error code asked for. The whole-core app +// must now run and be validly placed, and so must the fragmenters. This is +// what turns needs_repack from a label into a checked claim: if the workload +// still could not run, the node lied to the controller. +// +// Test params +// ----------- +// - HYPERVISOR. Skipped under Kubevirt, where concrete CPU selection belongs +// to the kubelet rather than to the pillar allocator this test exercises. +// +// Suite placement +// --------------- +// - TestAppsSuite, with the other CPU placement tests: it wants the same device +// (8 CPUs, 2 threads per core), so the VM can be reused. +func TestCPUPlacementNeedsRepack(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + if hypervisor == evetest.HypervisorKubevirt { + evetestT.Skip("under Kubevirt the kubelet selects CPUs, not the pillar allocator") + } + + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + MinCPUs: 8, + ThreadsPerCore: 2, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + + // Checked before anything is deployed: without SMT siblings a core cannot be + // half-owned at all, so there is no fragmentation to create and every + // assertion below would either hold vacuously or fail for the wrong reason. + topo, err := device.HostCPUTopology() + t.Expect(err).ToNot(HaveOccurred()) + if !deviceHasSMT(topo) { + evetestT.Skip("the device has no SMT sibling threads, so no physical core can " + + "be left half-owned and a node cannot be fragmented at all") + } + cores := physicalCores(topo) + evetest.Logger().Infof("device physical cores (as sibling lists): %v", cores) + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // Phases 2 and 3 read the node's CPU pool report, which only reaches the + // controller on the periodic ZInfoDevice publish; at the 10 minute default + // the test would spend most of its time waiting for a message. + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.DevInfoInterval, placementDevInfoInterval) + devConfig.SetConfigProperties(cfgProps) + + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := addLocalNI(devConfig) + + // Phase 1: the workloads that can fragment the node. + fragmenters := make([]*placedApp, 0, repackFragmenters) + for i := 0; i < repackFragmenters; i++ { + spec := threadGranularApp(fmt.Sprintf("cpu-frag-app-%d", i+1), + repackFragmenterVCPUs, appSSHFwdPort+uint16(i)) + appUUID := devConfig.AddApplication(placementAppConfig(spec, niUUID, 0)) + fragmenters = append(fragmenters, &placedApp{spec: spec, uuid: appUUID}) + } + device.ApplyConfig(devConfig, true, true) + evetest.Checkpoint("fragmenters-configured") + + waitUntilAppsRunning(device, fragmenters) + logPlacementDiagnostics(device, appUUIDs(fragmenters), placementShellTimeout) + for _, app := range fragmenters { + assertThreadGranularPlacement(t, device, app) + } + // Their sets must be disjoint and EVE must still have somewhere to run: + // everything below reasons about which CPUs are taken, so it is worth + // nothing if two workloads were handed the same CPU. + assertPlacementSetInvariants(t, device, topo, fragmenters) + evetest.Checkpoint("fragmenters-placed") + + // The node's own account while all three run. Also the source of the + // EVE-reserved CPU set: the housekeeping pool's CPUs that are not reported + // free are exactly the ones held back for EVE, which is knowledge the + // simulation below needs and which no test should hard-code. + running := awaitCPUPoolReport(t, device, + "all three thread-granular workloads holding their threads", + func(g Gomega, report cpuPoolReport) { + for _, app := range fragmenters { + g.Expect(report.dedicated.GetCpuIds()).To(ContainElements(intsOf(app.dedicated)), + "the node's dedicated pool %v omits host CPUs %v, which %q holds", + report.dedicated.GetCpuIds(), app.dedicated, app.spec.appName) + } + }) + reserved := subtractCPUs(running.housekeeping.GetCpuIds(), + running.housekeeping.GetFreeCpuIds()) + evetest.Logger().Infof("CPUs held back for EVE itself: %v", reserved) + + // Phase 2: stop one fragmenter to open the gaps. Which one is derived from + // the allocation the device made, because the thread-granular allocator's + // choice of CPUs is not the test's to dictate: releasing the wrong pair + // would hand back a whole free core and there would be nothing to prove. + victim, fragmented := chooseFragmentingVictim(t, cores, reserved, fragmenters) + evetest.Logger().Infof("stopping %q (host CPUs %v) to fragment the node: "+ + "expecting free threads %v, half-owned cores %v and no free whole core", + victim.spec.appName, victim.dedicated, fragmented.free, fragmented.halfOwned) + setPlacementAppsActivated(t, device, devConfig, niUUID, []*placedApp{victim}, false) + + var survivors []*placedApp + for _, app := range fragmenters { + if app != victim { + survivors = append(survivors, app) + } + } + + // The state the rest of the test depends on, read from the report a + // controller would use to decide whether another workload fits here. Free + // threads above the whole-core request while no whole core is free is the + // interesting condition: a controller counting free threads would answer + // "yes, it fits" and be wrong. + fragmentedReport := awaitCPUPoolReport(t, device, + "free threads but not one free whole core", + func(g Gomega, report cpuPoolReport) { + g.Expect(report.housekeeping.GetFreeCpuIds()).To(ContainElements(intsOf(victim.dedicated)), + "host CPUs %v released by the stopped %q are not reported as free "+ + "(free housekeeping CPUs: %v)", victim.dedicated, + victim.spec.appName, report.housekeeping.GetFreeCpuIds()) + g.Expect(report.housekeeping.GetFreeWholeCores()).To(BeZero(), + "the node reports %d free whole core(s) with free threads %v; the "+ + "thread-granular workloads were sized to leave a thread of every "+ + "allocatable core taken, so a free whole core means the node was "+ + "never fragmented and nothing below would be testing the repack "+ + "verdict", report.housekeeping.GetFreeWholeCores(), + report.housekeeping.GetFreeCpuIds()) + }) + freeThreads := fragmentedReport.housekeeping.GetFreeCpuIds() + t.Expect(len(freeThreads)).To(BeNumerically(">=", repackWholeCoreVCPUs), + "the node reports only %d free thread(s) (%v), fewer than the %d vCPUs the "+ + "whole-core workload asks for; the interesting failure is the one where "+ + "a plain free-thread count says the workload fits -- with fewer free "+ + "threads than vCPUs even a naive count would refuse it, and the "+ + "whole-core accounting would not be what made the difference", + len(freeThreads), freeThreads, repackWholeCoreVCPUs) + // Fragmentation, stated as such: a core with one thread taken and one free is + // the thing that cannot arise from whole-core workloads, and the reason the + // free threads above are unusable for a whole-core request. + halfOwned := halfOwnedCores(cores, fragmentedReport.dedicated.GetCpuIds(), freeThreads) + t.Expect(halfOwned).ToNot(BeEmpty(), + "no physical core has one thread taken and the other free (dedicated %v, "+ + "free %v); without a half-owned core the node is merely full, which is a "+ + "genuine insufficiency rather than the repackable shortage this test is "+ + "about", fragmentedReport.dedicated.GetCpuIds(), freeThreads) + evetest.Logger().Infof("node fragmented: free threads %v, free whole cores %d, "+ + "half-owned cores %v (dedicated %v)", freeThreads, + fragmentedReport.housekeeping.GetFreeWholeCores(), halfOwned, + fragmentedReport.dedicated.GetCpuIds()) + evetest.Checkpoint("node-fragmented") + + // Phase 3: the whole-core workload the node cannot place right now, but could + // place if the fragmenters were restarted alongside it. + wholeCoreSpec := wholeCoreSMTApp("cpu-repack-core-app", repackWholeCoreVCPUs, + appSSHFwdPort+repackFragmenters) + wholeCore := &placedApp{spec: wholeCoreSpec} + wholeCore.uuid = devConfig.AddApplication(placementAppConfig(wholeCoreSpec, niUUID, 0)) + // Subscribed before the config is applied, so a failure reported quickly + // cannot be missed. + coreUpdates, stopCoreWatch := device.WatchAppInfo(wholeCore.uuid) + defer stopCoreWatch() + device.ApplyConfig(devConfig, true, true) + + reported := awaitAppErrors(t, coreUpdates, wholeCoreSpec.appName, repackErrorTimeout) + codes := errorCodesOf(reported) + t.Expect(codes).To(ContainElement(errCodeNeedsRepack), + "%q could not be placed on a node with free threads %v and no free whole "+ + "core, and reported %v; the planned layout for the configured set does "+ + "fit this node, so this is a shortage a repack would fix and the node "+ + "must say so with %q -- a controller told %q has no reason to attempt "+ + "the one remedy it has", + wholeCoreSpec.appName, freeThreads, codes, errCodeNeedsRepack, errCodeInsufficient) + t.Expect(codes).ToNot(ContainElement(errCodeInsufficient), + "%q reported %q alongside %v; the two codes are alternatives -- one says a "+ + "repack would fix this and the other that nothing would -- so reporting "+ + "both leaves the controller no verdict at all", + wholeCoreSpec.appName, errCodeInsufficient, codes) + logPlacementDiagnostics(device, + append(appUUIDs(fragmenters), wholeCore.uuid), placementShellTimeout) + + // A failed placement holds nothing and is rated as nothing. The quality + // channel describes running workloads: a "needs repack" quality here would + // have the device publish the "running, but could be packed better" advisory + // beside a fatal error, telling the controller a workload is up when it is + // not. + failedStatus, err := readDomainCPUStatus(device, wholeCore.uuid) + t.Expect(err).ToNot(HaveOccurred()) + t.Expect(failedStatus.CPUs).To(BeEmpty(), + "%q failed to start, so it must hold no host CPUs, but domainmgr records %v", + wholeCoreSpec.appName, failedStatus.CPUs) + t.Expect(failedStatus.PlacementQuality).To(Equal(placementQualityUnspecified), + "%q never started, so it has no placement to rate, but its quality is "+ + "reported as %s", wholeCoreSpec.appName, + placementQualityName(failedStatus.PlacementQuality)) + t.Expect(device.GetAppInfo(wholeCore.uuid).GetState()). + ToNot(Equal(eveinfo.ZSwState_RUNNING), + "%q is reported as running although the device refused to place it", + wholeCoreSpec.appName) + + // And the node's account is unchanged: a refused workload must not have + // consumed the capacity it was refused. + awaitCPUPoolReport(t, device, "the refused workload holding nothing", + func(g Gomega, report cpuPoolReport) { + g.Expect(report.housekeeping.GetFreeCpuIds()).To(ConsistOf(intsOf(freeThreads)), + "the free housekeeping CPUs changed from %v to %v although %q was "+ + "refused and never ran", freeThreads, + report.housekeeping.GetFreeCpuIds(), wholeCoreSpec.appName) + }) + evetest.Checkpoint("needs-repack-reported") + + // Phase 4: the other code. A request for more physical cores than the device + // has cannot be satisfied by any arrangement, so no repack helps and the node + // must not suggest one. Deployed after the verdict above is recorded, and + // left in place afterwards: a workload the plan cannot place takes no CPUs + // from anyone, so it changes nothing for the others. + oversizedSpec := onePerCoreApp("cpu-repack-oversized-app", repackOversizedVCPUs, + appSSHFwdPort+repackFragmenters+1) + oversized := &placedApp{spec: oversizedSpec} + oversized.uuid = devConfig.AddApplication(placementAppConfig(oversizedSpec, niUUID, 0)) + oversizedUpdates, stopOversizedWatch := device.WatchAppInfo(oversized.uuid) + defer stopOversizedWatch() + device.ApplyConfig(devConfig, true, true) + + oversizedCodes := errorCodesOf(awaitAppErrors(t, oversizedUpdates, + oversizedSpec.appName, repackErrorTimeout)) + t.Expect(oversizedCodes).To(ContainElement(errCodeInsufficient), + "%q asked for %d dedicated physical cores on a device with %d cores in "+ + "total and reported %v; no arrangement of workloads can satisfy that, so "+ + "the node must report %q and not send the controller repacking a node "+ + "that is not the problem", oversizedSpec.appName, repackOversizedVCPUs, + len(cores), oversizedCodes, errCodeInsufficient) + t.Expect(oversizedCodes).ToNot(ContainElement(errCodeNeedsRepack), + "%q asked for more cores than the device has and reported %q; a repack "+ + "cannot conjure cores, and telling the controller otherwise invites it "+ + "to restart every pinned workload on the node for nothing", + oversizedSpec.appName, errCodeNeedsRepack) + evetest.Checkpoint("insufficient-reported") + + // Phase 5: perform the repack the node asked for, and see whether it was + // telling the truth. Stopping the pinned workloads and starting them again + // together is exactly what needs_repack advises: the whole-core workload must + // now get its core, and the fragmenters must still fit around it -- that is + // what the plan claimed when the device chose the code. If the workload still + // cannot start, the node reported an actionable failure that no action fixes. + repacked := append([]*placedApp{wholeCore}, survivors...) + evetest.Logger().Infof("repacking: stopping and restarting %d pinned workload(s)", + len(repacked)) + setPlacementAppsActivated(t, device, devConfig, niUUID, repacked, false) + // Said out loud before the attempt, because a failure to come up here is not + // a flaky app start: it means the device reported an actionable failure that + // the prescribed action does not fix. + evetest.Logger().Infof("starting the pinned workloads together again; %q must now "+ + "be placed -- it reported %s, so a repack is exactly what should let it run", + wholeCoreSpec.appName, errCodeNeedsRepack) + setPlacementAppsActivated(t, device, devConfig, niUUID, repacked, true) + logPlacementDiagnostics(device, appUUIDs(repacked), placementShellTimeout) + + // Placement is verified in full rather than just "it started": a repack that + // brought the workload up without its whole core, or without the per-vCPU + // pinning, would have satisfied the promise in name only. Optimality is not + // required -- the workloads are released together but the device activates + // them in whatever order it reaches them, so whichever starts first may take + // a slot the plan set aside for another (see assertAppPlacement). + assertAppPlacement(t, device, topo, wholeCore, false) + for _, app := range survivors { + assertThreadGranularPlacement(t, device, app) + } + assertPlacementSetInvariants(t, device, topo, repacked) + evetest.Checkpoint("repack-honoured") + + for _, app := range append(repacked, victim, oversized) { + deleteAppAndWait(t, device, devConfig, app.uuid) + } +} + +// assertThreadGranularPlacement verifies one thread-granular dedicated workload: +// it holds host CPUs exclusively, but as individual logical CPUs rather than as +// whole physical cores. +// +// The negative half of this is the interesting half. No per-vCPU assignment and +// no synthesized guest SMT topology is what separates thread granularity from +// whole-core placement, and it is precisely why such a workload can leave a core +// half-owned: it never asks for the sibling. A regression that quietly promoted +// these workloads to whole-core placement would make the node impossible to +// fragment, and every fragmentation test would then pass by never reaching the +// state it means to test. +func assertThreadGranularPlacement(t *GomegaWithT, device *evetest.EdgeDevice, + app *placedApp) { + spec := app.spec + t.Eventually(func(g Gomega) { + status, err := readDomainCPUStatus(device, app.uuid) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(status.CPUs).ToNot(BeEmpty(), + "domainmgr recorded no CPU set at all for %s", spec.appName) + app.status = status + }, placementSettleTimeout, placementPolling).Should(Succeed()) + + status := app.status + app.dedicated = subtractCPUs(status.CPUs, status.EmulatorCPUs) + evetest.Logger().Infof("thread-granular placement for %q: dedicated host CPUs %v "+ + "(emulator CPUs %v), pinned=%v, quality=%s", spec.appName, app.dedicated, + status.EmulatorCPUs, status.CPUsPinned, + placementQualityName(status.PlacementQuality)) + + t.Expect(status.CPUsPinned).To(BeTrue(), + "a dedicated CPU policy must give %q host CPUs of its own even without "+ + "full_pcpus_only, and must do so without the legacy pin_cpu flag", + spec.appName) + t.Expect(app.dedicated).To(HaveLen(spec.vCPUs), + "%q asked for %d vCPUs at thread granularity, so it must hold exactly that "+ + "many host CPUs, not %v", spec.appName, spec.vCPUs, app.dedicated) + t.Expect(uniqueCPUs(app.dedicated)).To(HaveLen(spec.vCPUs), + "the same host CPU must not be counted twice for %q (dedicated: %v)", + spec.appName, app.dedicated) + t.Expect(status.OrderedCPUs).To(BeEmpty(), + "%q asked for dedicated CPUs without full_pcpus_only, so no vCPU may be "+ + "bound to a fixed host CPU (recorded: %v); pinning per vCPU here would "+ + "mean the workload was given whole-core treatment it did not ask for", + spec.appName, status.OrderedCPUs) + t.Expect(status.VMTopology.Threads).To(BeZero(), + "%q holds individual SMT threads, which nothing guarantees are siblings, so "+ + "it must keep the flat guest topology; a synthesized threads=%d would "+ + "tell the guest its vCPUs share a core when they may not", + spec.appName, status.VMTopology.Threads) + t.Expect(status.PlacementQuality).To(Equal(placementQualityUnspecified), + "%q is not whole-core placed, so there is no whole-core layout to rate it "+ + "against and its quality must not be reported as %s", spec.appName, + placementQualityName(status.PlacementQuality)) + + // The kernel enforces it as a cpuset covering exactly those CPUs, and the + // vCPUs float within it -- which is all thread granularity promises. Both are + // read inside one Eventually so they cannot describe different moments. + t.Eventually(func(g Gomega) { + cpuset, err := device.AppCPUSet(app.uuid) + g.Expect(err).ToNot(HaveOccurred()) + affinities, err := device.AppVCPUAffinities(app.uuid) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(affinities).To(HaveLen(spec.vCPUs)) + g.Expect(cpuset).To(ConsistOf(intsOf(app.dedicated)), + "the cpuset %v of %q must be exactly its dedicated CPUs %v: wider and it "+ + "runs on CPUs the node considers free, narrower and it cannot use "+ + "what it was given", cpuset, spec.appName, app.dedicated) + for vcpu, allowed := range affinities { + g.Expect(allowed).To(ConsistOf(intsOf(app.dedicated)), + "guest vCPU %d of %q may run on %v while the workload holds %v; a "+ + "thread-granular workload's vCPUs are confined to its own CPUs and "+ + "free to move within them", vcpu, spec.appName, allowed, app.dedicated) + } + app.cpuset = cpuset + }, placementSettleTimeout, placementPolling).Should(Succeed()) + evetest.Logger().Infof("cpuset for %q: %v", spec.appName, app.cpuset) +} + +// nodeShape is what a set of dedicated CPUs leaves of the node's physical cores, +// computed from the topology rather than read from EVE -- so it can be used to +// predict the outcome of stopping a workload before stopping it. +type nodeShape struct { + // free is every logical CPU that is neither dedicated to a workload nor held + // back for EVE, i.e. what could still be handed out. + free []uint32 + // freeWhole is one representative CPU per physical core all of whose threads + // are free. This -- not len(free) -- bounds how many more whole-core + // workloads fit. + freeWhole []uint32 + // halfOwned is one representative CPU per physical core with at least one + // thread taken and at least one free. Such a core is the fingerprint of + // thread-granular allocation, and is refused to every whole-core request. + halfOwned []uint32 +} + +// shapeOf classifies every physical core against a dedicated and a reserved set. +func shapeOf(cores [][]uint32, dedicated, reserved []uint32) nodeShape { + isDedicated := cpuSetOf(dedicated) + isReserved := cpuSetOf(reserved) + var shape nodeShape + for _, core := range cores { + var taken, free []uint32 + for _, cpu := range core { + switch { + case isDedicated[cpu]: + taken = append(taken, cpu) + case !isReserved[cpu]: + free = append(free, cpu) + } + } + shape.free = append(shape.free, free...) + switch { + case len(free) == len(core): + shape.freeWhole = append(shape.freeWhole, core[0]) + case len(taken) > 0 && len(free) > 0: + shape.halfOwned = append(shape.halfOwned, core[0]) + } + } + return shape +} + +// chooseFragmentingVictim picks the thread-granular workload whose stopping +// leaves the node fragmented: no free whole core, yet more free threads than a +// whole-core request needs. +// +// It is a choice rather than a constant because the thread-granular allocator +// decides which CPUs each workload gets, and the workloads activate +// concurrently, so which of them ended up holding which pair is not the test's +// to dictate. Releasing the wrong pair hands back a complete free core and the +// whole-core workload below would simply start -- proving nothing. Simulating +// each candidate against the observed allocation is what makes the fragmented +// state reachable on every run instead of two runs in three. +func chooseFragmentingVictim(t *GomegaWithT, cores [][]uint32, reserved []uint32, + apps []*placedApp) (*placedApp, nodeShape) { + var all []uint32 + for _, app := range apps { + all = append(all, app.dedicated...) + } + var best *placedApp + var bestShape nodeShape + for _, app := range apps { + shape := shapeOf(cores, subtractCPUs(all, app.dedicated), reserved) + evetest.Logger().Infof("stopping %q (host CPUs %v) would leave free threads %v, "+ + "free whole cores %v, half-owned cores %v", app.spec.appName, app.dedicated, + shape.free, shape.freeWhole, shape.halfOwned) + if len(shape.freeWhole) > 0 || len(shape.halfOwned) == 0 || + len(shape.free) < repackWholeCoreVCPUs { + continue + } + // Prefer the candidate that half-owns the most cores: that is the + // starkest form of the state -- every allocatable core carrying one + // thread of somebody's workload and one thread nobody can use. + if best == nil || len(shape.halfOwned) > len(bestShape.halfOwned) { + best, bestShape = app, shape + } + } + t.Expect(best).ToNot(BeNil(), + "no thread-granular workload could be stopped to leave the node fragmented: "+ + "the ones deployed hold %v of a node whose cores are %v (reserved for "+ + "EVE: %v), and releasing any one of them either frees a whole core or "+ + "leaves too few free threads. The sizing above (see "+ + "repackFragmenterVCPUs) assumes a device with four dual-thread cores, "+ + "one CPU of which EVE reserves; on a differently sized device it has to "+ + "be recomputed", all, cores, reserved) + return best, bestShape +} + +// setPlacementAppsActivated flips the Activate flag of the given applications in +// one configuration change and waits until the device reports each of them in +// the state that change asks for. +// +// The local configuration is the authority here rather than +// EdgeDevice.ActivateApplication: the test adds applications to it between these +// calls, and mixing the two would silently re-activate a workload that was +// deliberately stopped. +func setPlacementAppsActivated(t *GomegaWithT, device *evetest.EdgeDevice, + devConfig *evetest.EdgeDeviceConfig, niUUID uuid.UUID, + apps []*placedApp, activate bool) { + updates := make([]<-chan *eveinfo.ZInfoApp, 0, len(apps)) + for _, app := range apps { + // Subscribed before the config change: the transition must not be missed + // while the device is applying it. + appUpdates, stop := device.WatchAppInfo(app.uuid) + defer stop() + updates = append(updates, appUpdates) + + config := placementAppConfig(app.spec, niUUID, 0) + config.Activate = activate + devConfig.UpdateApplication(app.uuid, config) + } + device.ApplyConfig(devConfig, true, true) + + for i, app := range apps { + if activate { + device.WaitUntilAppIsRunning(app.uuid, placementRunningTimeout) + continue + } + // HALTED is the device's statement that the workload is down and has + // given its CPUs back, which is what the phases here depend on -- a + // config the device merely received would not do. + t.Eventually(updates[i], repackStopTimeout, placementPolling).Should( + Receive(matchers.SatisfyPredicate( + "the application is reported as stopped", + func(info *eveinfo.ZInfoApp) bool { + return info.GetState() == eveinfo.ZSwState_HALTED + })), + "%q was deactivated but the device never reported it as halted", + app.spec.appName) + } +} + +// awaitAppErrors blocks until the device reports at least one error for the +// application and returns the errors it reported. +// +// Read from the app's info message rather than from any device-internal state, +// because the question this test asks -- can a controller tell the two failures +// apart? -- is only meaningful about what actually reaches the controller. +func awaitAppErrors(t *GomegaWithT, updates <-chan *eveinfo.ZInfoApp, + appName string, timeout time.Duration) []*eveinfo.ErrorInfo { + var reported []*eveinfo.ErrorInfo + t.Eventually(updates, timeout, placementPolling).Should(Receive( + matchers.SatisfyPredicate("the device reports an error for "+appName, + func(info *eveinfo.ZInfoApp) bool { + if len(info.GetAppErr()) == 0 { + return false + } + reported = info.GetAppErr() + return true + })), + "the device never reported any error for %q, although it cannot place it", + appName) + for _, appErr := range reported { + evetest.Logger().Infof("%q reported error: code=%q severity=%s retry=%q "+ + "description=%q", appName, appErr.GetErrorCode(), appErr.GetSeverity(), + appErr.GetRetryCondition(), appErr.GetDescription()) + } + return reported +} + +// errorCodesOf lists the machine-parseable codes of the reported errors. An +// error without a code is reported as the empty string rather than dropped, so +// "the failure carried no code at all" shows up in the assertion message. +func errorCodesOf(errs []*eveinfo.ErrorInfo) []string { + codes := make([]string, 0, len(errs)) + for _, appErr := range errs { + codes = append(codes, appErr.GetErrorCode()) + } + return codes +} + +// halfOwnedCores returns one representative CPU per physical core with at least +// one thread dedicated to a workload and at least one thread free, derived from +// the node's own pool report so that the report and the topology are checked +// against each other rather than the topology alone. +func halfOwnedCores(cores [][]uint32, dedicated, free []uint32) []uint32 { + isDedicated := cpuSetOf(dedicated) + isFree := cpuSetOf(free) + var out []uint32 + for _, core := range cores { + var taken, spare int + for _, cpu := range core { + if isDedicated[cpu] { + taken++ + } + if isFree[cpu] { + spare++ + } + } + if taken > 0 && spare > 0 { + out = append(out, core[0]) + } + } + return out +} + +// physicalCores groups the device's logical CPUs into physical cores, each as +// its ascending sibling list, cores ordered by their lowest sibling. +func physicalCores(topo evetest.HostTopology) [][]uint32 { + var cores [][]uint32 + seen := map[uint32]bool{} + for _, cpu := range topo.IDs() { + if seen[cpu] { + continue + } + siblings := topo.SiblingsOf(cpu) + if len(siblings) == 0 { + siblings = []uint32{cpu} + } + for _, sibling := range siblings { + seen[sibling] = true + } + cores = append(cores, siblings) + } + return cores +} + +func cpuSetOf(cpus []uint32) map[uint32]bool { + set := make(map[uint32]bool, len(cpus)) + for _, cpu := range cpus { + set[cpu] = true + } + return set +} diff --git a/evetest/tests/apps/testsuite_test.go b/evetest/tests/apps/testsuite_test.go index 5693edb3d23..143b1c1d900 100644 --- a/evetest/tests/apps/testsuite_test.go +++ b/evetest/tests/apps/testsuite_test.go @@ -95,6 +95,10 @@ import ( // unused is consumed by it, not spare: no other workload gets it in its // cpuset, the node does not advertise it as free, and nothing is ever // observed running on it. +// - TestCPUPlacementNeedsRepack -- on a node fragmented by thread-granular +// workloads (free threads, no free whole core) a whole-core app is refused +// with cpu.placement.needs_repack rather than insufficient, and a repack +// really does let it run. // - TestVMAppPurgeReplacesVMIRS -- a plain purge of a healthy app leaves // exactly one VMIRS, named for the new generation. Kubevirt only; skips // on any other hypervisor. @@ -158,6 +162,11 @@ func TestAppsSuite(test *testing.T) { evetest.TestCase{ Test: TestCPUPlacementParkedSiblings, }, + // Also the 8-CPU, 2-threads-per-core device: it needs SMT siblings to + // fragment the node in the first place. + evetest.TestCase{ + Test: TestCPUPlacementNeedsRepack, + }, evetest.TestCase{ Test: TestVMAppPurgeReplacesVMIRS, }, From 8e861b7aabf281d1bad4932e63099b0cde24c446 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 13:06:23 +0000 Subject: [PATCH 14/15] evetest: e2e test that unsatisfiable placement requests fail closed A request that cannot be honoured must be refused, not approximated. Silently falling back to a weaker placement would hand back a running workload whose timing guarantees are gone, with nothing in the reported state to say so -- the worst outcome available, because it looks like success. Each class of unsatisfiable request is checked separately with its own error code, since a controller needs to distinguish a request that is malformed from one the node cannot support from one it merely has no room for. The workload must never boot, and the node's dedicated CPU pool must be unchanged afterwards: a refused request that leaked cores would shrink the node's capacity with every retry. The refusal also has to persist. A placement failure that healed itself as soon as some unrelated workload released cores would start the workload at an arbitrary moment with a placement nobody validated, so this asserts the workload stays down until its config changes -- the same way EVE already treats a workload whose PCI device is unavailable. Signed-off-by: Mikhail Malyshev --- evetest/tests/apps/cpufailsclosed_test.go | 777 ++++++++++++++++++++++ evetest/tests/apps/testsuite_test.go | 9 + 2 files changed, 786 insertions(+) create mode 100644 evetest/tests/apps/cpufailsclosed_test.go diff --git a/evetest/tests/apps/cpufailsclosed_test.go b/evetest/tests/apps/cpufailsclosed_test.go new file mode 100644 index 00000000000..9774e864887 --- /dev/null +++ b/evetest/tests/apps/cpufailsclosed_test.go @@ -0,0 +1,777 @@ +// Copyright (c) 2026 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Test that a CPU placement request this node cannot honour is refused with the +// right structured code, and that the refusal costs the node nothing. + +package apps_test + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + // revive:disable:dot-imports + . "github.com/onsi/gomega" + + uuid "github.com/satori/go.uuid" + + eveconfig "github.com/lf-edge/eve-api/go/config" + "github.com/lf-edge/eve-api/go/evecommon" + eveinfo "github.com/lf-edge/eve-api/go/info" + "github.com/lf-edge/eve/evetest" + "github.com/lf-edge/eve/evetest/matchers" + "github.com/lf-edge/eve/evetest/netmodels" + pillartypes "github.com/lf-edge/eve/pkg/pillar/types" +) + +// The remaining CPU placement failure codes as they travel on the wire, in +// ZInfoApp.AppErr[].error_code. errCodeNeedsRepack and errCodeInsufficient -- +// the two a shortage produces -- live in cpuneedsrepack_test.go. +// +// Spelled out here rather than imported from pillar's types package for the same +// two reasons given there: evetest builds against a released pillar module, +// which does not carry constants added on a feature branch; and these strings +// *are* the published contract a controller matches on, so a test referencing +// the device's own constant could not notice one of them being renamed under it. +const ( + // errCodeOddVCPU means whole-core-SMT was asked for with an odd vCPU count. + // Every core contributes two vCPUs, so no arrangement satisfies it and no + // amount of free capacity would help. + errCodeOddVCPU = "cpu.policy.odd_vcpu" + // errCodeTierUnavailable means the requested isolation tier cannot be + // provided by this node as it is currently running. + errCodeTierUnavailable = "cpu.isolation.tier_unavailable" + // errCodePolicyInvalid means the policy itself is malformed or asks for + // something this device does not implement. + errCodePolicyInvalid = "cpu.policy.invalid" + // errCodeTopologyUnsupported means the active hypervisor cannot bind vCPUs + // to named host CPUs at all. Listed only so the exclusivity check below + // covers it; it is not reachable on the kvm path this test runs on (see the + // test's doc comment). + errCodeTopologyUnsupported = "cpu.topology.unsupported" + // errCodeDegraded is the advisory a *running* workload gets. It is in the + // exclusivity list because emitting it beside a fatal refusal would tell the + // controller a workload is up when it never started. + errCodeDegraded = "cpu.placement.degraded" +) + +// placementErrorCodes is the whole published registry of cpu.* codes. Each +// refusal below is required to carry its own code and none of the others, which +// is what makes the assertion "exactly this code" rather than "at least this +// code": a device that answered every unsatisfiable request with, say, +// cpu.policy.invalid would satisfy a per-case ContainElement check for that one +// case while being useless to a controller. +var placementErrorCodes = []string{ + errCodeOddVCPU, + errCodeTierUnavailable, + errCodePolicyInvalid, + errCodeTopologyUnsupported, + errCodeNeedsRepack, + errCodeInsufficient, + errCodeDegraded, +} + +const ( + // refusalHolderVCPUs is the whole-core-SMT request of the one application in + // this test that must actually run: two vCPUs, i.e. exactly one physical + // core. It is there so that "the dedicated pool is unchanged" is a statement + // about a non-empty pool -- an assertion that the pool stayed empty would + // hold on a device that never allocates anything at all. + refusalHolderVCPUs = 2 + // refusalOddVCPUs is the odd vCPU count offered to whole-core-SMT. Three + // rather than one: a single vCPU could be refused by an allocator that simply + // cannot round up, whereas three is a count the device *could* serve by + // handing over two whole cores and wasting a thread -- which is precisely the + // silent downgrade the code exists to prevent. + refusalOddVCPUs = 3 + // refusalEvenVCPUs is the vCPU count used by every case whose defect is not + // the count itself. Even, and small enough to fit the free capacity left + // after the holder, so no refusal below can be explained away as a shortage. + refusalEvenVCPUs = 2 + // refusalInvalidThreadsPerCore is a threads_per_core the API has no meaning + // for. Only 1 (sibling parked) and 2 (both siblings become vCPUs) exist. + refusalInvalidThreadsPerCore = 3 + // refusalErrorTimeout bounds waiting for the device to report a refusal. It + // covers creating the app's volume from the already-downloaded image, the + // trip through zedmanager/domainmgr and the info message back. + refusalErrorTimeout = 10 * time.Minute + // refusalPoolStableFor is how long the node's CPU pool report must keep + // matching the pre-attempt baseline, and refusalPoolPolling how often it is + // re-read. + // + // Checked with Consistently rather than Eventually because the property is + // "nothing changed": a single read could be satisfied by a message published + // before the refused applications were ever configured. Spanning several + // publish intervals (the device is lowered to placementDevInfoInterval) + // guarantees at least one report computed *after* the refusals is seen. + // + // Neither the baseline nor the re-reads touch the device: the pool report is + // picked out of the ZInfoDevice messages the harness already receives, so the + // whole stability window costs no SSH sessions. + refusalPoolStableFor = 100 * time.Second + refusalPoolPolling = 10 * time.Second +) + +// refusedRequest is one unsatisfiable placement request plus the code the device +// must answer it with. +type refusedRequest struct { + spec cpuPlacementApp + // wantCode is the error_code the controller must receive. + wantCode string + // why states what makes the request impossible to honour, and is quoted in + // the failure messages so a failing run explains itself. + why string +} + +// TestCPUPlacementFailsClosed verifies that every class of unsatisfiable CPU +// placement request this device can be sent is refused, is refused with its own +// machine-parseable code, and leaves the node exactly as it found it. +// +// Failing *open* is the failure mode that matters here, and it is the reason +// this test exists rather than trusting the unit tests that cover the same +// decisions. A device that starts a workload with weaker guarantees than it +// asked for reports success: the app is RUNNING, the controller is satisfied, +// and the workload silently runs without the isolation it was deployed for. That +// is worse than not starting, and it is invisible from the controller -- so the +// only place it can be caught is a test that sends the impossible request over +// the wire and looks at what comes back (design doc §3 "fail cleanly, never +// mis-place", §10.4 "a hard requirement never degrades silently"). +// +// The second half of each verdict is that a refusal must be *free*. A request +// the device rejects must not have reserved the cores it was rejected for: the +// controller was told the workload did not start, so from its point of view that +// capacity is available, and a node quietly holding it would shrink its own +// usable capacity on every rejected deploy -- with nothing anywhere saying why. +// That is asserted against the node's own cpu_pools report, which is what a +// controller actually reads to answer "will the next workload fit here?". +// +// Codes covered, and what makes each request impossible +// ---------------------------------------------------- +// - cpu.policy.odd_vcpu -- whole-core-SMT with an odd vCPU count. Each +// dedicated core contributes both its SMT threads as vCPUs, so the vCPU +// count is necessarily even and no arrangement of cores satisfies an odd +// one. Free capacity is irrelevant. +// - cpu.isolation.tier_unavailable -- isolation tier "hard". Shedding kernel +// housekeeping off the workload's cores needs a kernel command-line change +// and a reboot, which the device cannot do while placing a workload. The +// alternative to refusing is delivering soft isolation and calling it hard. +// - cpu.policy.invalid, twice, by the two independent routes that reach it: +// threads_per_core = 3, a value the API gives no meaning to (only 1 and 2 +// exist); and disruption policy "protect", which was deliberately turned +// from accepted-but-unenforced into a refusal, because nothing on the device +// defers a node-level action yet and a workload told it is shielded would +// still be taken down by a reboot without warning. +// +// Not covered: cpu.topology.unsupported. It fires when the active hypervisor +// cannot pin individual vCPUs or synthesize a guest SMT topology, which is a +// property of the hypervisor (kvm can, kubevirt cannot -- hypervisor.Capabilities +// .CPUTopologyPinning), not of anything a controller can configure. On the kvm +// path this test runs on there is no configuration that reaches it, and this test +// runs only on kvm because under kubevirt the kubelet, not the pillar allocator, +// selects CPUs. It is listed in placementErrorCodes anyway, so that a device +// answering one of the cases below with it would be caught. +// +// Network model +// ------------- +// - netmodels.SingleEthWithDHCP -- placement is not network dependent; a +// single mgmt+apps port is enough to run the holder app and reach it. +// +// Device configuration +// -------------------- +// - SystemAdapter for eth0 (DHCP, mgmt+apps), local NI "local-ni". +// - timer.deviceinfo.interval lowered to its minimum: the node's CPU pool +// report rides the periodic ZInfoDevice publish, and the residue assertion +// needs a report computed after the refusals. +// - 8 CPUs as 4 dual-thread cores. EVE reserves the lowest CPU, making its +// whole core unallocatable, so three cores are allocatable: one goes to the +// holder app and two stay free. Every refused request below asks for at most +// two cores, so each of them would fit the free capacity if it were +// honourable -- which is what makes the refusal attributable to the policy +// rather than to a shortage. +// - Five container apps (lfedge/evetest-ubuntu-ctr): one that must run and +// four that must not, all on the same boot. A refused app costs no CPUs, so +// they cost no capacity and there is no reason to spend a device boot each. +// +// Phases / assertions +// ------------------- +// 1. holder-placed: the one valid app runs and is fully verified, and the +// node's dedicated pool and free housekeeping set are recorded as the +// baseline the refusals must not move. +// 2. refusals-reported: the four unsatisfiable apps are deployed together. Each +// must report its own code and none of the other cpu.* codes, at ERROR +// severity with a non-empty retry condition, and must not reach RUNNING or +// even BOOTING. Asserted entirely from the info messages the controller +// receives, which touches the device not at all. +// 3. no-residue: the node's dedicated pool and free housekeeping set still +// equal the baseline across several publish intervals, and -- read from the +// device in a single batched session -- no refused app is activated, holds a +// host CPU or carries a placement quality, while the holder still holds +// exactly what it held. A refusal must not be paid for by a workload that +// was already placed, nor by the node's capacity. +// +// The device-side reads are deliberately batched into one SSH session rather +// than one per file. By the time this test runs, most of the node's CPUs belong +// to a pinned workload and EVE's own services are squeezed onto the +// housekeeping set; short-lived SSH sessions on such a device were observed +// dying with "connection reset by peer". Ten small reads are ten chances to lose +// one, and retrying each of them just spends more sessions -- so the chattiness +// is removed instead of worked around. +// +// Test params +// ----------- +// - HYPERVISOR. Skipped under Kubevirt, where concrete CPU selection belongs +// to the kubelet rather than to the pillar allocator this test exercises. +// +// Suite placement +// --------------- +// - TestAppsSuite, with the other CPU placement tests: it wants the same +// device (8 CPUs, 2 threads per core), so the VM can be reused. +func TestCPUPlacementFailsClosed(test *testing.T) { + evetestT := evetest.Init(test) + t := NewGomegaWithT(evetestT) + defer evetest.Close() + + evetest.DefineTestParameters( + evetest.HypervisorParameter(), + ) + hypervisor := evetest.GetHypervisorParameterValue() + if hypervisor == evetest.HypervisorKubevirt { + evetestT.Skip("under Kubevirt the kubelet selects CPUs, not the pillar allocator") + } + + evetest.Setup( + evetest.RequireEdgeDevice{ + Name: devName, + MinCPUs: 8, + ThreadsPerCore: 2, + WithHypervisor: hypervisor, + DeviceReusePolicy: evetest.ResetDeviceConfig, + }, + evetest.RequireNetworkModel{ + NetworkModel: netmodels.SingleEthWithDHCP, + }, + ) + device := evetest.GetEdgeDevice(devName) + + // The holder app asks for a whole core with both siblings as vCPUs, which a + // device of single-thread cores cannot provide. Without it there is no + // non-empty dedicated pool to assert stays unchanged, so the interesting half + // of this test would hold vacuously. + topo, err := device.HostCPUTopology() + t.Expect(err).ToNot(HaveOccurred()) + if !deviceHasSMT(topo) { + evetestT.Skip("the device has no SMT sibling threads, so the whole-core-SMT " + + "holder application cannot be placed and there would be no non-empty " + + "dedicated pool for the refusals to leave unchanged") + } + evetest.Checkpoint("setup-done") + + devConfig := evetest.NewEdgeDeviceConfig(devName) + + // The residue assertion reads the node's CPU pool report, which only reaches + // the controller on the periodic ZInfoDevice publish; at the 10 minute default + // the test would spend most of its time waiting for a message. + cfgProps := pillartypes.NewConfigItemValueMap() + cfgProps.SetGlobalValueInt(pillartypes.DevInfoInterval, placementDevInfoInterval) + devConfig.SetConfigProperties(cfgProps) + + dhcpNet := devConfig.AddNetwork(evetest.DHCPNetworkConfig{ + NetworkType: evecommon.NetworkType_V4Only, + }) + devConfig.AddNetworkAdapter(evetest.NetworkAdapterConfig{ + LogicalLabel: "ethernet0", + PhysicalLabel: "eth0", + InterfaceName: "eth0", + NetworkUUID: dhcpNet, + Usage: evecommon.PhyIoMemberUsage_PhyIoUsageMgmtAndApps, + }) + niUUID := addLocalNI(devConfig) + + // Phase 1: the workload that must be unaffected by everything below. + holderSpec := wholeCoreSMTApp("cpu-refused-holder-app", refusalHolderVCPUs, + appSSHFwdPort) + holder := &placedApp{spec: holderSpec} + holder.uuid = devConfig.AddApplication(placementAppConfig(holderSpec, niUUID, 0)) + device.ApplyConfig(devConfig, true, true) + evetest.Checkpoint("holder-configured") + + device.WaitUntilAppIsRunning(holder.uuid, placementRunningTimeout) + logPlacementDiagnostics(device, []uuid.UUID{holder.uuid}, placementShellTimeout) + // Nothing else is running, so the holder must have landed on its planned slot. + assertAppPlacement(t, device, topo, holder, true) + holderOrdered := append([]uint32(nil), holder.status.OrderedCPUs...) + holderDedicated := append([]uint32(nil), holder.dedicated...) + + // The baseline every refusal must leave alone, taken from the node's own + // report rather than from domainmgr's per-app view: this is the report a + // controller consults to decide whether the next workload fits here, so it is + // the one whose silent shrinking would do the damage. + baseline := awaitCPUPoolReport(t, device, + "the holder application holding its whole core", + func(g Gomega, report cpuPoolReport) { + g.Expect(report.dedicated.GetCpuIds()).To(ContainElements(intsOf(holderDedicated)), + "the node's dedicated pool %v omits host CPUs %v, which %q holds", + report.dedicated.GetCpuIds(), holderDedicated, holderSpec.appName) + }) + baseDedicated := append([]uint32(nil), baseline.dedicated.GetCpuIds()...) + baseFree := append([]uint32(nil), baseline.housekeeping.GetFreeCpuIds()...) + baseFreeWholeCores := baseline.housekeeping.GetFreeWholeCores() + evetest.Logger().Infof("baseline before any refused request: dedicated %v, free "+ + "housekeeping %v, free whole cores %d", baseDedicated, baseFree, baseFreeWholeCores) + // Stated as an assertion because every refusal below is only attributable to + // its policy if the request could otherwise have been served. With no free + // whole core left, "the device refused it" would prove nothing. + t.Expect(baseFreeWholeCores).To(BeNumerically(">=", 2), + "the node reports only %d free whole core(s) after placing %q; the refused "+ + "requests below ask for up to two cores each and must be refusable only "+ + "on policy grounds, so the node has to have the capacity to serve them. "+ + "The sizing above assumes four dual-thread cores, one CPU of which EVE "+ + "reserves; on a differently sized device it has to be recomputed", + baseFreeWholeCores, holderSpec.appName) + evetest.Checkpoint("holder-placed") + + // Phase 2: every class of request this node cannot honour, deployed together. + // Each is sized to fit the free capacity, so nothing here can be refused for + // running out of cores. + refused := []refusedRequest{ + { + spec: oddVCPUWholeCoreApp("cpu-refused-odd-vcpu-app", refusalOddVCPUs, + appSSHFwdPort+1), + wantCode: errCodeOddVCPU, + why: fmt.Sprintf("whole-core-smt turns both SMT threads of every "+ + "dedicated core into vCPUs, so the vCPU count is necessarily even; "+ + "%d cannot be produced by any number of cores", refusalOddVCPUs), + }, + { + spec: hardIsolationApp("cpu-refused-hard-tier-app", refusalEvenVCPUs, + appSSHFwdPort+2), + wantCode: errCodeTierUnavailable, + why: "hard isolation sheds kernel housekeeping off the workload's cores, " + + "which needs a kernel command-line change and a reboot; the device " + + "cannot do that while placing a workload, and delivering soft " + + "isolation instead would hand the workload weaker guarantees than it " + + "asked for without telling anyone", + }, + { + spec: invalidThreadsPerCoreApp("cpu-refused-threads-app", refusalEvenVCPUs, + appSSHFwdPort+3), + wantCode: errCodePolicyInvalid, + why: fmt.Sprintf("threads_per_core=%d has no meaning in the API: a "+ + "dedicated core either contributes one vCPU with its sibling parked "+ + "or both siblings as vCPUs, and nothing else", + refusalInvalidThreadsPerCore), + }, + { + spec: protectedDisruptionApp("cpu-refused-protect-app", refusalEvenVCPUs, + appSSHFwdPort+4), + wantCode: errCodePolicyInvalid, + why: "nothing on the device defers a node-level disruptive action yet, so " + + "accepting \"protect\" would tell the controller its workload is " + + "shielded while a reboot or an upgrade still takes it down unannounced", + }, + } + + // Subscribed before the configuration is applied, so a refusal reported + // quickly cannot be missed. + watched := make([]<-chan *eveinfo.ZInfoApp, 0, len(refused)) + uuids := make([]uuid.UUID, 0, len(refused)) + for _, request := range refused { + appUUID := devConfig.AddApplication(placementAppConfig(request.spec, niUUID, 0)) + uuids = append(uuids, appUUID) + updates, stop := device.WatchAppInfo(appUUID) + defer stop() + watched = append(watched, updates) + } + device.ApplyConfig(devConfig, true, true) + evetest.Checkpoint("refusals-configured") + + for i, request := range refused { + appName := request.spec.appName + reported := awaitRefusalError(t, watched[i], appName, refusalErrorTimeout) + codes := errorCodesOf(reported) + + // The whole point: the controller must receive this code, not merely an + // error. Asserted on error_code and never on the description -- the prose + // is for humans and may be reworded at any time, while the code is the + // published contract. + t.Expect(codes).To(ContainElement(request.wantCode), + "%q asked for something this node cannot honour (%s) and reported %v; "+ + "the controller must receive %q so it can tell the user what to fix "+ + "instead of showing a bare \"failed to start\"", + appName, request.why, codes, request.wantCode) + // And no other code from the registry, so each condition is genuinely + // distinguished rather than all of them collapsing onto one token. + for _, other := range placementErrorCodes { + if other == request.wantCode { + continue + } + t.Expect(codes).ToNot(ContainElement(other), + "%q must be refused with %q alone but also reported %q (all codes: "+ + "%v); the codes are alternatives a controller switches on, so "+ + "reporting two of them leaves it no verdict", + appName, request.wantCode, other, codes) + } + + // Severity, because the code alone does not say whether the workload is + // broken or merely imperfect. cpu.placement.degraded rides the same + // per-app error list at WARNING severity precisely so a controller can + // tell an advisory from a refusal; a refusal published at WARNING would be + // read as "running, with a remark". + refusal := errorWithCode(reported, request.wantCode) + t.Expect(refusal).ToNot(BeNil()) + t.Expect(refusal.GetSeverity()).To(Equal(eveinfo.Severity_SEVERITY_ERROR), + "%q was refused with %q at severity %s; a refusal is not an advisory, "+ + "and a controller that distinguishes them by severity would treat "+ + "this workload as running", appName, request.wantCode, + refusal.GetSeverity()) + + // The code says which condition it is; the retry condition says what would + // change the answer. Every refusal here is fail-closed, so there is always + // something true to say -- either the workload's configuration or the node + // has to change -- and an empty field would leave the operator with an + // error they cannot act on. Only its presence is asserted: the wording is + // prose meant for a human and will be rewritten. + t.Expect(refusal.GetRetryCondition()).ToNot(BeEmpty(), + "%q was refused with %q but the device suggested no retry condition; "+ + "the refusal is permanent until something changes, so the one field "+ + "that could tell the operator *what* to change must not be empty", + appName, request.wantCode) + + evetest.Logger().Infof("%q refused with code=%q severity=%s retry=%q "+ + "description=%q", appName, refusal.GetErrorCode(), refusal.GetSeverity(), + refusal.GetRetryCondition(), refusal.GetDescription()) + + // Fail closed. This is the assertion the whole test is built around: a + // workload that got weaker guarantees than it asked for and started anyway + // looks like success from every angle a controller can see. + // + // The states are checked rather than the API's ZSwState_ERROR because no + // pillar agent puts an application into that state for a domainmgr + // failure -- the reported state stays at the last one actually reached + // (INSTALLED, since the domain is never created). What "fail closed" means + // operationally is that the workload never ran, so that is what is + // asserted: never RUNNING and never even BOOTING, alongside the + // ERROR-severity refusal above. Phase 3 adds the device's own verdict, + // that zedmanager never considered it activated. + state := device.GetAppInfo(uuids[i]).GetState() + evetest.Logger().Infof("%q is reported in state %s after being refused", + appName, state) + t.Expect(state).ToNot(Equal(eveinfo.ZSwState_RUNNING), + "%q is reported as RUNNING although the device refused its placement "+ + "(%s); starting the workload with weaker guarantees than it asked "+ + "for is the one outcome worse than not starting it, because nothing "+ + "anywhere says it happened", appName, request.why) + t.Expect(state).ToNot(Equal(eveinfo.ZSwState_BOOTING), + "%q is reported as BOOTING although its placement was refused; the "+ + "domain must never be created for a request the device cannot honour", + appName) + } + logPlacementDiagnostics(device, append([]uuid.UUID{holder.uuid}, uuids...), + placementShellTimeout) + evetest.Checkpoint("refusals-reported") + + // Phase 3: the refusals cost nothing. A rejected request that still reserved + // cores would shrink the node's usable capacity on every failed deploy, and + // the controller -- which was told the workload did not start -- would have no + // way of learning where the cores went. + // + // The node's own account comes first, because it is what a controller sizes + // the next deploy against. Held to the baseline across several publish + // intervals rather than read once, so a report computed before the refused + // apps existed cannot satisfy it. + t.Consistently(func(g Gomega) { + report, err := readCPUPoolReport(device) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(report.dedicated.GetCpuIds()).To(ConsistOf(intsOf(baseDedicated)), + "the node's dedicated pool changed from %v to %v although every "+ + "application deployed since was refused and never ran; a refused "+ + "request that still reserves cores silently shrinks the node's "+ + "capacity, and nothing tells the controller why", + baseDedicated, report.dedicated.GetCpuIds()) + g.Expect(report.housekeeping.GetFreeCpuIds()).To(ConsistOf(intsOf(baseFree)), + "the free housekeeping CPUs changed from %v to %v although every "+ + "application deployed since was refused", baseFree, + report.housekeeping.GetFreeCpuIds()) + g.Expect(report.housekeeping.GetFreeWholeCores()).To(Equal(baseFreeWholeCores), + "the node reported %d free whole core(s) before the refused requests "+ + "and %d after; the whole-core count is what bounds how many more "+ + "whole-core workloads fit here, so losing one to a refused request "+ + "costs the node a workload it could have run", baseFreeWholeCores, + report.housekeeping.GetFreeWholeCores()) + }, refusalPoolStableFor, refusalPoolPolling).Should(Succeed()) + + // And the device's own books, for every workload at once. The pool report + // above is an aggregate; this is where a refused workload individually + // holding something would show up, and it is also where the holder is checked + // to be untouched -- refusing a request by disturbing a workload that was + // already placed would be a different way of making the same mistake, landing + // the cost of an impossible ask on somebody who asked for something possible. + residues := readRefusalResidues(t, device, + append([]uuid.UUID{holder.uuid}, uuids...)) + + for i, request := range refused { + appName := request.spec.appName + residue := residues[uuids[i].String()] + // zedmanager's own verdict on whether it brought the workload up. The + // device-side counterpart of "did not reach RUNNING": the reported state + // says what the controller was told, this says what the device believes it + // did. The refusal has to stop the activation, not accompany it. + t.Expect(residue.App).ToNot(BeNil(), + "zedmanager published no AppInstanceStatus for %q at all", appName) + t.Expect(residue.App.Activated).To(BeFalse(), + "zedmanager considers %q activated although domainmgr refused to place "+ + "it (%s)", appName, request.why) + // domainmgr does publish a DomainStatus for a workload it refused to + // place -- that is where the error itself lives -- so this is expected to + // be present, and its emptiness is the assertion. + t.Expect(residue.Domain).ToNot(BeNil(), + "domainmgr published no DomainStatus for %q, so what it recorded for "+ + "the refused workload cannot be checked", appName) + t.Expect(residue.Domain.CPUs).To(BeEmpty(), + "%q was refused and never ran, but domainmgr records host CPUs %v for "+ + "it; cores held by a workload the controller believes did not start "+ + "are cores nobody can ever use again", appName, residue.Domain.CPUs) + t.Expect(residue.Domain.OrderedCPUs).To(BeEmpty(), + "%q was refused but domainmgr recorded a per-vCPU assignment %v for it", + appName, residue.Domain.OrderedCPUs) + t.Expect(residue.Domain.PlacementQuality).To(Equal(placementQualityUnspecified), + "%q never ran, so it has no placement to rate, but its quality is "+ + "reported as %s; a quality here would have the device publish the "+ + "\"running, but could be packed better\" advisory beside a fatal "+ + "error", appName, placementQualityName(residue.Domain.PlacementQuality)) + } + + t.Expect(device.GetAppInfo(holder.uuid).GetState()).To(Equal(eveinfo.ZSwState_RUNNING), + "%q was running before the refused applications were deployed and must "+ + "still be running", holderSpec.appName) + holderNow := residues[holder.uuid.String()].Domain + t.Expect(holderNow).ToNot(BeNil(), + "domainmgr no longer publishes a DomainStatus for the running application %q", + holderSpec.appName) + t.Expect(holderNow.OrderedCPUs).To(Equal(holderOrdered), + "%q ran on host CPUs %v before the refused requests and runs on %v after; "+ + "a running workload is never moved, least of all on account of a "+ + "request that was rejected", holderSpec.appName, holderOrdered, + holderNow.OrderedCPUs) + t.Expect(subtractCPUs(holderNow.CPUs, holderNow.EmulatorCPUs)). + To(ConsistOf(intsOf(holderDedicated)), + "%q occupied host CPUs %v exclusively before the refused requests and "+ + "occupies %v after", holderSpec.appName, holderDedicated, + subtractCPUs(holderNow.CPUs, holderNow.EmulatorCPUs)) + evetest.Checkpoint("no-residue") + + for _, appUUID := range append([]uuid.UUID{holder.uuid}, uuids...) { + deleteAppAndWait(t, device, devConfig, appUUID) + } +} + +// oddVCPUWholeCoreApp asks for whole-core-SMT placement with an odd vCPU count. +// The policy is exactly the valid whole-core-SMT one -- only the count is wrong +// -- so the refusal cannot be attributed to anything else in the policy. +func oddVCPUWholeCoreApp(appName string, vCPUs int, sshFwdPort uint16) cpuPlacementApp { + spec := wholeCoreSMTApp(appName, vCPUs, sshFwdPort) + spec.mode = "whole-core-smt (odd vCPU count)" + return spec +} + +// hardIsolationApp asks for the hard isolation tier on top of an otherwise valid +// whole-core-SMT request. +// +// The tier is the only defect: the mode, the thread count and the vCPU count are +// all satisfiable, and the request fits the node's free capacity. So if the +// device starts this workload it has delivered soft isolation under the name of +// hard isolation -- the silent downgrade the tier check exists to prevent. +func hardIsolationApp(appName string, vCPUs int, sshFwdPort uint16) cpuPlacementApp { + spec := wholeCoreSMTApp(appName, vCPUs, sshFwdPort) + spec.mode = "whole-core-smt + hard isolation tier" + spec.placement.IsolationTier = eveconfig.IsolationTier_ISOLATION_TIER_HARD + return spec +} + +// invalidThreadsPerCoreApp asks for a threads-per-core the API defines no +// meaning for. A device that quietly rounded it to 1 or 2 would place the +// workload in a mode the controller never asked for. +// +// The harness copies every CPUPlacementConfig field straight into VmConfig +// without validating or clamping it (evetest.CPUPlacementConfig -> toProto), so +// this really does reach the device as threads_per_core=3 -- as do the hard +// isolation tier and the protect disruption policy below. That is deliberate: +// the device has to be the backstop, since a real controller is not obliged to +// pre-validate and a test whose harness sanitized the request would prove +// nothing about the device. +func invalidThreadsPerCoreApp(appName string, vCPUs int, + sshFwdPort uint16) cpuPlacementApp { + spec := wholeCoreSMTApp(appName, vCPUs, sshFwdPort) + spec.mode = fmt.Sprintf("dedicated whole cores, threads_per_core=%d", + refusalInvalidThreadsPerCore) + spec.placement.ThreadsPerCore = refusalInvalidThreadsPerCore + return spec +} + +// protectedDisruptionApp asks for the "protect" disruption policy on top of an +// otherwise valid whole-core-SMT request. +// +// This one is a refusal by deliberate choice rather than by impossibility: the +// placement itself is fine, but nothing on the device defers a node-level +// disruptive action, so accepting the field would promise the workload a +// protection it does not have. It is here to keep that choice from silently +// regressing into acceptance -- which is what "unimplemented field is ignored" +// normally decays into. +func protectedDisruptionApp(appName string, vCPUs int, sshFwdPort uint16) cpuPlacementApp { + spec := wholeCoreSMTApp(appName, vCPUs, sshFwdPort) + spec.mode = "whole-core-smt + protect disruption policy" + spec.placement.DisruptionPolicy = eveconfig.DisruptionPolicy_DISRUPTION_POLICY_PROTECT + return spec +} + +// awaitRefusalError blocks until the device reports an error carrying a +// machine-parseable code for the application, and returns every error it +// reported alongside it. +// +// It waits for a *coded* error rather than for any error, because an application +// on its way to being placed can briefly carry unrelated errors from the volume +// or network path, and the first of those arriving would otherwise be mistaken +// for the refusal and fail the code assertion for the wrong reason. A refusal +// that never carries a code at all still fails here -- on the timeout, with the +// errors that were reported quoted in the message, which is the correct verdict: +// an uncoded refusal is exactly the failure this test exists to catch. +func awaitRefusalError(t *GomegaWithT, updates <-chan *eveinfo.ZInfoApp, + appName string, timeout time.Duration) []*eveinfo.ErrorInfo { + var reported, lastSeen []*eveinfo.ErrorInfo + t.Eventually(updates, timeout, placementPolling).Should(Receive( + matchers.SatisfyPredicate("the device reports a coded error for "+appName, + func(info *eveinfo.ZInfoApp) bool { + if len(info.GetAppErr()) > 0 { + lastSeen = info.GetAppErr() + } + for _, appErr := range info.GetAppErr() { + if appErr.GetErrorCode() != "" { + reported = info.GetAppErr() + return true + } + } + return false + })), + "the device never reported an error with a machine-parseable code for %q, "+ + "although it cannot honour its placement request; the errors it did "+ + "report were %v. A refusal a controller can only understand by reading "+ + "English prose is not usable: it cannot tell the user what to fix", + appName, lastSeen) + for _, appErr := range reported { + evetest.Logger().Infof("%q reported error: code=%q severity=%s retry=%q "+ + "description=%q", appName, appErr.GetErrorCode(), appErr.GetSeverity(), + appErr.GetRetryCondition(), appErr.GetDescription()) + } + return reported +} + +// errorWithCode returns the reported error carrying the given code, so severity +// and retry condition are read off the refusal itself rather than off whichever +// error happens to come first. +func errorWithCode(errs []*eveinfo.ErrorInfo, code string) *eveinfo.ErrorInfo { + for _, appErr := range errs { + if appErr.GetErrorCode() == code { + return appErr + } + } + return nil +} + +// refusalResidue is everything the device itself records about one workload that +// bears on whether a refusal cost anything: what domainmgr assigned it, and +// whether zedmanager considers it brought up. +// +// Both members are pointers so that "the device published nothing for this +// workload" is distinguishable from "it published an empty allocation". The +// difference matters: the second is the expected outcome of a refusal, while the +// first would mean the assertions below were checking a file that does not +// exist and passing for that reason. +type refusalResidue struct { + Domain *domainCPUStatus `json:"domain"` + App *struct { + Activated bool + } `json:"app"` +} + +// refusalResidueScript prints one JSON object holding, for each requested +// workload, domainmgr's DomainStatus and zedmanager's AppInstanceStatus -- the +// two pubsub files under /run that say what the device actually did. +// +// It assembles the reply itself instead of the test reading the files one by +// one, because by this point most of the node's CPUs are dedicated to a pinned +// workload and EVE's services share what is left; a short-lived SSH session on +// such a device has been seen dying with "connection reset by peer". Ten reads +// are ten chances to lose one, and per-read retries only spend more sessions -- +// so this is one session for the whole set. +// +// Missing files become JSON null rather than an error: the caller distinguishes +// them, and a shell failing halfway would return unparseable output that says +// nothing about which file was the problem. +const refusalResidueScript = `printf '{' +sep='' +for u in @UUIDS@; do + printf '%s"%s":{"domain":' "$sep" "$u" + if [ -f "/run/domainmgr/DomainStatus/$u.json" ]; then + cat "/run/domainmgr/DomainStatus/$u.json" + else + printf 'null' + fi + printf ',"app":' + if [ -f "/run/zedmanager/AppInstanceStatus/$u.json" ]; then + cat "/run/zedmanager/AppInstanceStatus/$u.json" + else + printf 'null' + fi + printf '}' + sep=',' +done +printf '}' +` + +// readRefusalResidues runs that script and returns the result keyed by +// application UUID. +// +// Retried only to absorb an SSH session that failed to come up -- none of the +// values it returns converges, so a device that really did leak a core returns +// the same wrong answer every time and still fails the assertions. +func readRefusalResidues(t *GomegaWithT, device *evetest.EdgeDevice, + appUUIDs []uuid.UUID) map[string]refusalResidue { + + ids := make([]string, 0, len(appUUIDs)) + for _, appUUID := range appUUIDs { + ids = append(ids, appUUID.String()) + } + script := strings.ReplaceAll(refusalResidueScript, "@UUIDS@", strings.Join(ids, " ")) + + residues := map[string]refusalResidue{} + t.Eventually(func(g Gomega) { + stdout, stderr, err := device.RunShellScript(script, placementShellTimeout, 0) + g.Expect(err).ToNot(HaveOccurred(), + "failed to read the pubsub state of %v (stderr: %s)", ids, stderr) + g.Expect(json.Unmarshal([]byte(stdout), &residues)).To(Succeed(), + "the device's reply is not the expected JSON document: %q", stdout) + g.Expect(residues).To(HaveLen(len(ids))) + }, placementSettleTimeout, placementPolling).Should(Succeed(), + "the device never returned the recorded CPU state of %v", ids) + + // Logged for every workload before anything is asserted, so a failing run + // shows what the device recorded for all of them, not only for the first one + // that broke an assertion. + for _, id := range ids { + residue := residues[id] + if residue.Domain == nil { + evetest.Logger().Infof("device state for %s: no DomainStatus published "+ + "(activated=%v)", id, residue.App != nil && residue.App.Activated) + continue + } + evetest.Logger().Infof("device state for %s: activated=%v cpus=%v "+ + "emulator=%v ordered=%v quality=%s", id, + residue.App != nil && residue.App.Activated, residue.Domain.CPUs, + residue.Domain.EmulatorCPUs, residue.Domain.OrderedCPUs, + placementQualityName(residue.Domain.PlacementQuality)) + } + return residues +} diff --git a/evetest/tests/apps/testsuite_test.go b/evetest/tests/apps/testsuite_test.go index 143b1c1d900..75a27baf188 100644 --- a/evetest/tests/apps/testsuite_test.go +++ b/evetest/tests/apps/testsuite_test.go @@ -99,6 +99,10 @@ import ( // workloads (free threads, no free whole core) a whole-core app is refused // with cpu.placement.needs_repack rather than insufficient, and a repack // really does let it run. +// - TestCPUPlacementFailsClosed -- every class of unsatisfiable placement +// request (odd vCPUs with whole-core SMT, hard isolation tier, invalid +// policy) is refused with its own error_code at ERROR severity, never boots, +// and leaves the node's dedicated CPU pool unchanged. // - TestVMAppPurgeReplacesVMIRS -- a plain purge of a healthy app leaves // exactly one VMIRS, named for the new generation. Kubevirt only; skips // on any other hypervisor. @@ -167,6 +171,11 @@ func TestAppsSuite(test *testing.T) { evetest.TestCase{ Test: TestCPUPlacementNeedsRepack, }, + // Same 8-CPU, 2-threads-per-core device: the whole-core-SMT holder + // application it keeps running throughout needs real SMT siblings. + evetest.TestCase{ + Test: TestCPUPlacementFailsClosed, + }, evetest.TestCase{ Test: TestVMAppPurgeReplacesVMIRS, }, From b3ce4ed88299179e504d8d58ec340f02f980c5c2 Mon Sep 17 00:00:00 2001 From: Mikhail Malyshev Date: Mon, 17 Aug 2026 17:34:30 +0000 Subject: [PATCH 15/15] pillar, evetest: build against the CPU-placement eve-api [DO NOT MERGE] The device-side code in this branch needs API that has not landed upstream yet: the VmConfig CPU placement fields, the CPU topology and capability reporting on device info, ZInfoDevice.cpu_pools, API_CAPABILITY_CPU_PLACEMENT_POLICY and ErrorInfo.error_code. Without them nothing here compiles, so this commit points pillar and evetest at the fork carrying the proposed API (lf-edge/eve-api#155) through a replace directive, and vendors it. This commit exists only so the branch can be built, run and reviewed while the API is under discussion. It must be dropped and replaced by an ordinary `make bump-eve-api` once the API lands in lf-edge/eve-api: a replace directive pointing at a personal fork breaks dependency tracking and SBOM/licensing, and is never acceptable on master. Signed-off-by: Mikhail Malyshev --- evetest/go.mod | 2 + evetest/go.sum | 4 +- pkg/pillar/go.mod | 2 + pkg/pillar/go.sum | 4 +- .../lf-edge/eve-api/go/config/vm.pb.go | 526 ++- .../lf-edge/eve-api/go/info/common.pb.go | 91 +- .../lf-edge/eve-api/go/info/hardware.pb.go | 951 ++++- .../lf-edge/eve-api/go/info/info.pb.go | 3697 +++++++++-------- pkg/pillar/vendor/modules.txt | 3 +- 9 files changed, 3302 insertions(+), 1978 deletions(-) diff --git a/evetest/go.mod b/evetest/go.mod index 2357bf2dee0..adb38040bf8 100644 --- a/evetest/go.mod +++ b/evetest/go.mod @@ -100,3 +100,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace github.com/lf-edge/eve-api/go => github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc diff --git a/evetest/go.sum b/evetest/go.sum index 0a3087be766..3aee45fb5fb 100644 --- a/evetest/go.sum +++ b/evetest/go.sum @@ -107,8 +107,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/lf-edge/eve-api/go v0.0.0-20260812180240-99d02ddcfcb0 h1:vFrZPWDaFupy4K1fYIRDJOnW8aaGNMZ1/1DkNRXrZe0= -github.com/lf-edge/eve-api/go v0.0.0-20260812180240-99d02ddcfcb0/go.mod h1:6HxNA/qKJVEqwpuOFkcQ0h3QyotvAs/cjHLC961FPOY= github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d h1:tUBb9M6u42LXwHAYHyh22wJeUUQlTpDkXwRXalpRqbo= github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d/go.mod h1:Nn3juMJJ1G8dyHOebdZyS4jOB/fuxAd5fIajBaWjHr8= github.com/lf-edge/eve/pkg/pillar v0.0.0-20260421125048-8d3825045e4e h1:MGhvM4TrXHYyssTTcEkpc79zatoZ1oJRrrbjTU++ps0= @@ -149,6 +147,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc h1:8NwHnhH0EmLCnnicKa00cHP0maYbryuy4Up11il0I9A= +github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc/go.mod h1:6HxNA/qKJVEqwpuOFkcQ0h3QyotvAs/cjHLC961FPOY= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= diff --git a/pkg/pillar/go.mod b/pkg/pillar/go.mod index 1e3fd59058b..e29bb40dc3b 100644 --- a/pkg/pillar/go.mod +++ b/pkg/pillar/go.mod @@ -321,3 +321,5 @@ replace ( k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.33.5 k8s.io/sample-controller => k8s.io/sample-controller v0.33.5 ) + +replace github.com/lf-edge/eve-api/go => github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc diff --git a/pkg/pillar/go.sum b/pkg/pillar/go.sum index 65f1374e75a..4830741767b 100644 --- a/pkg/pillar/go.sum +++ b/pkg/pillar/go.sum @@ -497,8 +497,6 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lf-edge/edge-containers v0.0.0-20260502192833-bdbe764faf59 h1:fuyDqWUHuck0owe+PxJ4ey4jtOEpN0xZ0Y4k8qxx8AY= github.com/lf-edge/edge-containers v0.0.0-20260502192833-bdbe764faf59/go.mod h1:DoK51ifLgVh2GQ+IXTvmkhTuf8THijPCwYlNAbs2Nz0= -github.com/lf-edge/eve-api/go v0.0.0-20260812180240-99d02ddcfcb0 h1:vFrZPWDaFupy4K1fYIRDJOnW8aaGNMZ1/1DkNRXrZe0= -github.com/lf-edge/eve-api/go v0.0.0-20260812180240-99d02ddcfcb0/go.mod h1:6HxNA/qKJVEqwpuOFkcQ0h3QyotvAs/cjHLC961FPOY= github.com/lf-edge/eve-libs v0.0.0-20260807152845-882c84170868 h1:hGgskJxVKjCaiPv9zHVjOAe8ye+6HqGzdNUGEWy3n4c= github.com/lf-edge/eve-libs v0.0.0-20260807152845-882c84170868/go.mod h1:ZwkS/Xlx3klDLUobHesq3dAjiRfUlz0Pk6LxiRnhOck= github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d h1:tUBb9M6u42LXwHAYHyh22wJeUUQlTpDkXwRXalpRqbo= @@ -725,6 +723,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc h1:8NwHnhH0EmLCnnicKa00cHP0maYbryuy4Up11il0I9A= +github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc/go.mod h1:6HxNA/qKJVEqwpuOFkcQ0h3QyotvAs/cjHLC961FPOY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/config/vm.pb.go b/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/config/vm.pb.go index 6fe7a5d180d..5369fa799e8 100644 --- a/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/config/vm.pb.go +++ b/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/config/vm.pb.go @@ -86,6 +86,305 @@ func (VmMode) EnumDescriptor() ([]byte, []int) { return file_config_vm_proto_rawDescGZIP(), []int{0} } +// CpuPolicy selects how a workload's vCPUs are placed on host CPUs. +// Vocabulary follows the Kubernetes CPUManager model so the same intent +// translates to both the eve-kvm and eve-k (KubeVirt) variants. +type CpuPolicy int32 + +const ( + // Not set: legacy behavior. If pin_cpu is true the workload is treated + // as CPU_POLICY_DEDICATED with default allocation; otherwise shared. + CpuPolicy_CPU_POLICY_UNSPECIFIED CpuPolicy = 0 + // Best-effort placement in the shared pool (no pinning). + CpuPolicy_CPU_POLICY_SHARED CpuPolicy = 1 + // The workload gets host CPUs of its own; no other workload runs on + // them. Details are refined by full_pcpus_only / threads_per_core. + CpuPolicy_CPU_POLICY_DEDICATED CpuPolicy = 2 +) + +// Enum value maps for CpuPolicy. +var ( + CpuPolicy_name = map[int32]string{ + 0: "CPU_POLICY_UNSPECIFIED", + 1: "CPU_POLICY_SHARED", + 2: "CPU_POLICY_DEDICATED", + } + CpuPolicy_value = map[string]int32{ + "CPU_POLICY_UNSPECIFIED": 0, + "CPU_POLICY_SHARED": 1, + "CPU_POLICY_DEDICATED": 2, + } +) + +func (x CpuPolicy) Enum() *CpuPolicy { + p := new(CpuPolicy) + *p = x + return p +} + +func (x CpuPolicy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CpuPolicy) Descriptor() protoreflect.EnumDescriptor { + return file_config_vm_proto_enumTypes[1].Descriptor() +} + +func (CpuPolicy) Type() protoreflect.EnumType { + return &file_config_vm_proto_enumTypes[1] +} + +func (x CpuPolicy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CpuPolicy.Descriptor instead. +func (CpuPolicy) EnumDescriptor() ([]byte, []int) { + return file_config_vm_proto_rawDescGZIP(), []int{1} +} + +// NumaPolicy expresses how strictly a dedicated workload's CPUs (and +// memory) must be confined to a single NUMA node. Mirrors the Kubernetes +// TopologyManager policies. +type NumaPolicy int32 + +const ( + // Defaults to NUMA_POLICY_BEST_EFFORT. + NumaPolicy_NUMA_POLICY_UNSPECIFIED NumaPolicy = 0 + // No NUMA preference. + NumaPolicy_NUMA_POLICY_NONE NumaPolicy = 1 + // Prefer a single NUMA node; span nodes only if it does not fit. + NumaPolicy_NUMA_POLICY_BEST_EFFORT NumaPolicy = 2 + // Restricted placement: minimize the nodes spanned; fail-closed only + // if no placement exists at all. + NumaPolicy_NUMA_POLICY_RESTRICTED NumaPolicy = 3 + // Must fit in one NUMA node; the workload does not start otherwise. + NumaPolicy_NUMA_POLICY_SINGLE_NUMA_NODE NumaPolicy = 4 +) + +// Enum value maps for NumaPolicy. +var ( + NumaPolicy_name = map[int32]string{ + 0: "NUMA_POLICY_UNSPECIFIED", + 1: "NUMA_POLICY_NONE", + 2: "NUMA_POLICY_BEST_EFFORT", + 3: "NUMA_POLICY_RESTRICTED", + 4: "NUMA_POLICY_SINGLE_NUMA_NODE", + } + NumaPolicy_value = map[string]int32{ + "NUMA_POLICY_UNSPECIFIED": 0, + "NUMA_POLICY_NONE": 1, + "NUMA_POLICY_BEST_EFFORT": 2, + "NUMA_POLICY_RESTRICTED": 3, + "NUMA_POLICY_SINGLE_NUMA_NODE": 4, + } +) + +func (x NumaPolicy) Enum() *NumaPolicy { + p := new(NumaPolicy) + *p = x + return p +} + +func (x NumaPolicy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (NumaPolicy) Descriptor() protoreflect.EnumDescriptor { + return file_config_vm_proto_enumTypes[2].Descriptor() +} + +func (NumaPolicy) Type() protoreflect.EnumType { + return &file_config_vm_proto_enumTypes[2] +} + +func (x NumaPolicy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use NumaPolicy.Descriptor instead. +func (NumaPolicy) EnumDescriptor() ([]byte, []int) { + return file_config_vm_proto_rawDescGZIP(), []int{2} +} + +// IoPlacement selects where the hypervisor's non-vCPU threads (emulator, +// IO) of a dedicated workload run. +type IoPlacement int32 + +const ( + // Defaults to IO_PLACEMENT_DEDICATED. + IoPlacement_IO_PLACEMENT_UNSPECIFIED IoPlacement = 0 + // Emulator/IO threads stay within the workload's dedicated CPU set. + IoPlacement_IO_PLACEMENT_DEDICATED IoPlacement = 1 + // Emulator/IO threads are pinned off the dedicated cores onto the + // node's housekeeping set, so device emulation cannot steal cycles + // from busy (e.g. poll-mode) vCPUs. + IoPlacement_IO_PLACEMENT_HOUSEKEEPING IoPlacement = 2 +) + +// Enum value maps for IoPlacement. +var ( + IoPlacement_name = map[int32]string{ + 0: "IO_PLACEMENT_UNSPECIFIED", + 1: "IO_PLACEMENT_DEDICATED", + 2: "IO_PLACEMENT_HOUSEKEEPING", + } + IoPlacement_value = map[string]int32{ + "IO_PLACEMENT_UNSPECIFIED": 0, + "IO_PLACEMENT_DEDICATED": 1, + "IO_PLACEMENT_HOUSEKEEPING": 2, + } +) + +func (x IoPlacement) Enum() *IoPlacement { + p := new(IoPlacement) + *p = x + return p +} + +func (x IoPlacement) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (IoPlacement) Descriptor() protoreflect.EnumDescriptor { + return file_config_vm_proto_enumTypes[3].Descriptor() +} + +func (IoPlacement) Type() protoreflect.EnumType { + return &file_config_vm_proto_enumTypes[3] +} + +func (x IoPlacement) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use IoPlacement.Descriptor instead. +func (IoPlacement) EnumDescriptor() ([]byte, []int) { + return file_config_vm_proto_rawDescGZIP(), []int{3} +} + +// IsolationTier expresses how strongly a workload's CPUs must be shielded +// from interference. It is workload intent: the device derives whether the +// requested tier is achievable for this workload type from the node's +// granular capability report (info NodeCapabilities) and fails closed with +// a structured error if it is not. +type IsolationTier int32 + +const ( + // Defaults: ISOLATION_TIER_NONE for shared workloads, + // ISOLATION_TIER_SOFT for dedicated (pinned) workloads. + IsolationTier_ISOLATION_TIER_UNSPECIFIED IsolationTier = 0 + // No isolation beyond best-effort shared scheduling. + IsolationTier_ISOLATION_TIER_NONE IsolationTier = 1 + // cpuset pinning + SMT/NUMA-aware placement; no other workload runs on + // the workload's cores. Applied at runtime, no reboot required. + IsolationTier_ISOLATION_TIER_SOFT IsolationTier = 2 + // Kernel-level isolation (isolcpus/nohz_full/rcu_nocbs) on top of soft + // isolation; sheds kernel housekeeping from the cores. Requires a kernel + // command-line change and therefore a node reboot to apply. + IsolationTier_ISOLATION_TIER_HARD IsolationTier = 3 +) + +// Enum value maps for IsolationTier. +var ( + IsolationTier_name = map[int32]string{ + 0: "ISOLATION_TIER_UNSPECIFIED", + 1: "ISOLATION_TIER_NONE", + 2: "ISOLATION_TIER_SOFT", + 3: "ISOLATION_TIER_HARD", + } + IsolationTier_value = map[string]int32{ + "ISOLATION_TIER_UNSPECIFIED": 0, + "ISOLATION_TIER_NONE": 1, + "ISOLATION_TIER_SOFT": 2, + "ISOLATION_TIER_HARD": 3, + } +) + +func (x IsolationTier) Enum() *IsolationTier { + p := new(IsolationTier) + *p = x + return p +} + +func (x IsolationTier) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (IsolationTier) Descriptor() protoreflect.EnumDescriptor { + return file_config_vm_proto_enumTypes[4].Descriptor() +} + +func (IsolationTier) Type() protoreflect.EnumType { + return &file_config_vm_proto_enumTypes[4] +} + +func (x IsolationTier) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use IsolationTier.Descriptor instead. +func (IsolationTier) EnumDescriptor() ([]byte, []int) { + return file_config_vm_proto_rawDescGZIP(), []int{4} +} + +// DisruptionPolicy guards a running workload against collateral +// node-level disruptive actions (reboot/shutdown). +type DisruptionPolicy int32 + +const ( + // Defaults to DISRUPTION_POLICY_ALLOW. + DisruptionPolicy_DISRUPTION_POLICY_UNSPECIFIED DisruptionPolicy = 0 + // Node-level disruptive actions proceed normally. + DisruptionPolicy_DISRUPTION_POLICY_ALLOW DisruptionPolicy = 1 + // A node-level disruptive action that would take this running workload + // down is deferred and reported; the controller must re-issue the + // action with an explicit acknowledge to proceed. A targeted per-app + // restart is not deferred — it is itself the explicit decision. + DisruptionPolicy_DISRUPTION_POLICY_PROTECT DisruptionPolicy = 2 +) + +// Enum value maps for DisruptionPolicy. +var ( + DisruptionPolicy_name = map[int32]string{ + 0: "DISRUPTION_POLICY_UNSPECIFIED", + 1: "DISRUPTION_POLICY_ALLOW", + 2: "DISRUPTION_POLICY_PROTECT", + } + DisruptionPolicy_value = map[string]int32{ + "DISRUPTION_POLICY_UNSPECIFIED": 0, + "DISRUPTION_POLICY_ALLOW": 1, + "DISRUPTION_POLICY_PROTECT": 2, + } +) + +func (x DisruptionPolicy) Enum() *DisruptionPolicy { + p := new(DisruptionPolicy) + *p = x + return p +} + +func (x DisruptionPolicy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (DisruptionPolicy) Descriptor() protoreflect.EnumDescriptor { + return file_config_vm_proto_enumTypes[5].Descriptor() +} + +func (DisruptionPolicy) Type() protoreflect.EnumType { + return &file_config_vm_proto_enumTypes[5] +} + +func (x DisruptionPolicy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use DisruptionPolicy.Descriptor instead. +func (DisruptionPolicy) EnumDescriptor() ([]byte, []int) { + return file_config_vm_proto_rawDescGZIP(), []int{5} +} + // Boot mechanisms supported by the BIOS of the Virtual Machine type VmBootMode int32 @@ -120,11 +419,11 @@ func (x VmBootMode) String() string { } func (VmBootMode) Descriptor() protoreflect.EnumDescriptor { - return file_config_vm_proto_enumTypes[1].Descriptor() + return file_config_vm_proto_enumTypes[6].Descriptor() } func (VmBootMode) Type() protoreflect.EnumType { - return &file_config_vm_proto_enumTypes[1] + return &file_config_vm_proto_enumTypes[6] } func (x VmBootMode) Number() protoreflect.EnumNumber { @@ -133,7 +432,7 @@ func (x VmBootMode) Number() protoreflect.EnumNumber { // Deprecated: Use VmBootMode.Descriptor instead. func (VmBootMode) EnumDescriptor() ([]byte, []int) { - return file_config_vm_proto_rawDescGZIP(), []int{1} + return file_config_vm_proto_rawDescGZIP(), []int{6} } type VmConfig struct { @@ -157,11 +456,17 @@ type VmConfig struct { Rootdev string `protobuf:"bytes,7,opt,name=rootdev,proto3" json:"rootdev,omitempty"` Extraargs string `protobuf:"bytes,8,opt,name=extraargs,proto3" json:"extraargs,omitempty"` Bootloader string `protobuf:"bytes,9,opt,name=bootloader,proto3" json:"bootloader,omitempty"` - // Currently is not handled by EVE. + // Currently is not handled by EVE, and superseded — do not build on it. // CPU mask of the CPUs assigned to the VM. Represented in the form // "d[[,-]d]*". E.g. "0-2" or "0-2,5,6". CPUs start with 0. For example, the // mask "0,3" would mean that only physical CPUs 0 and 3 are available for // the VM. + // + // EVE has never parsed this field, and it is not planned to: naming concrete + // host CPUs is the device's responsibility, not the controller's. A workload + // expresses *what it needs* through the CPU placement policy below + // (cpu_policy and friends) and the device selects cores that satisfy it, + // topology-aware. Use cpu_policy instead. Cpus string `protobuf:"bytes,10,opt,name=cpus,proto3" json:"cpus,omitempty"` Devicetree string `protobuf:"bytes,11,opt,name=devicetree,proto3" json:"devicetree,omitempty"` Dtdev []string `protobuf:"bytes,12,rep,name=dtdev,proto3" json:"dtdev,omitempty"` @@ -182,6 +487,12 @@ type VmConfig struct { // for running QEMU threads are picked automatically by Pillar: it just takes // that amount of available physical CPUs that is defined with the 'vcpus' // parameter defined above. + // + // Superseded by cpu_policy below, which expresses HOW to pin (whole cores vs + // threads, NUMA strictness, where emulator/IO threads go, isolation) instead + // of only WHETHER to pin. Retained for backward compatibility: pin_cpu=true + // with cpu_policy unset is treated as CPU_POLICY_DEDICATED with default + // allocation; when cpu_policy is set it takes precedence over pin_cpu. PinCpu bool `protobuf:"varint,20,opt,name=pin_cpu,json=pinCpu,proto3" json:"pin_cpu,omitempty"` // Maximum amount of memory in kbytes allowed for VM monitor to occupy, // aka "overhead". E.g. for the qemu-kvm hypervisor the memory limit @@ -225,6 +536,31 @@ type VmConfig struct { // The setting is passed to OVMF via fw_cfg "opt/eve.bootorder" when the VM starts. // Changes take effect on the next VM restart. BootOrder evecommon.BootOrder `protobuf:"varint,27,opt,name=boot_order,json=bootOrder,proto3,enum=org.lfedge.eve.common.BootOrder" json:"boot_order,omitempty"` + // Pin or not; see enum. CPU_POLICY_DEDICATED activates the fields below. + CpuPolicy CpuPolicy `protobuf:"varint,28,opt,name=cpu_policy,json=cpuPolicy,proto3,enum=org.lfedge.eve.config.CpuPolicy" json:"cpu_policy,omitempty"` + // Require whole physical cores: no other workload runs on any SMT + // sibling of the workload's cores. Without it, allocation is at + // SMT-thread granularity and a physical core may be shared. + // Mirrors the Kubernetes CPUManager "full-pcpus-only" option. + FullPcpusOnly bool `protobuf:"varint,29,opt,name=full_pcpus_only,json=fullPcpusOnly,proto3" json:"full_pcpus_only,omitempty"` + // With full_pcpus_only, how many SMT siblings of each core become + // vCPUs: 2 = whole-core-SMT (both siblings are vCPUs, guest sees + // threads=2; requires an even vcpus count), 1 = one-per-core (sibling + // parked idle, guest sees threads=1). 0/unset defaults to 2 on SMT + // hardware. The guest is shown a truthful sockets/cores/threads + // topology and vCPUs are pinned 1:1, so guest sibling pairs correspond + // to real host sibling pairs. + ThreadsPerCore uint32 `protobuf:"varint,30,opt,name=threads_per_core,json=threadsPerCore,proto3" json:"threads_per_core,omitempty"` + // NUMA confinement for the dedicated CPUs; default best-effort. + NumaPolicy NumaPolicy `protobuf:"varint,31,opt,name=numa_policy,json=numaPolicy,proto3,enum=org.lfedge.eve.config.NumaPolicy" json:"numa_policy,omitempty"` + // Placement of hypervisor emulator/IO threads; default dedicated. + IoPlacement IoPlacement `protobuf:"varint,32,opt,name=io_placement,json=ioPlacement,proto3,enum=org.lfedge.eve.config.IoPlacement" json:"io_placement,omitempty"` + // Requested isolation tier; default soft for dedicated workloads. + // Requesting a tier this node/workload-type cannot satisfy fails closed + // with a structured error. + IsolationTier IsolationTier `protobuf:"varint,33,opt,name=isolation_tier,json=isolationTier,proto3,enum=org.lfedge.eve.config.IsolationTier" json:"isolation_tier,omitempty"` + // Protection against collateral node-level disruption; default allow. + DisruptionPolicy DisruptionPolicy `protobuf:"varint,34,opt,name=disruption_policy,json=disruptionPolicy,proto3,enum=org.lfedge.eve.config.DisruptionPolicy" json:"disruption_policy,omitempty"` } func (x *VmConfig) Reset() { @@ -448,6 +784,55 @@ func (x *VmConfig) GetBootOrder() evecommon.BootOrder { return evecommon.BootOrder(0) } +func (x *VmConfig) GetCpuPolicy() CpuPolicy { + if x != nil { + return x.CpuPolicy + } + return CpuPolicy_CPU_POLICY_UNSPECIFIED +} + +func (x *VmConfig) GetFullPcpusOnly() bool { + if x != nil { + return x.FullPcpusOnly + } + return false +} + +func (x *VmConfig) GetThreadsPerCore() uint32 { + if x != nil { + return x.ThreadsPerCore + } + return 0 +} + +func (x *VmConfig) GetNumaPolicy() NumaPolicy { + if x != nil { + return x.NumaPolicy + } + return NumaPolicy_NUMA_POLICY_UNSPECIFIED +} + +func (x *VmConfig) GetIoPlacement() IoPlacement { + if x != nil { + return x.IoPlacement + } + return IoPlacement_IO_PLACEMENT_UNSPECIFIED +} + +func (x *VmConfig) GetIsolationTier() IsolationTier { + if x != nil { + return x.IsolationTier + } + return IsolationTier_ISOLATION_TIER_UNSPECIFIED +} + +func (x *VmConfig) GetDisruptionPolicy() DisruptionPolicy { + if x != nil { + return x.DisruptionPolicy + } + return DisruptionPolicy_DISRUPTION_POLICY_UNSPECIFIED +} + var File_config_vm_proto protoreflect.FileDescriptor var file_config_vm_proto_rawDesc = []byte{ @@ -455,7 +840,7 @@ var file_config_vm_proto_rawDesc = []byte{ 0x6f, 0x12, 0x15, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x19, 0x65, 0x76, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x65, 0x76, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x22, 0xc2, 0x07, 0x0a, 0x08, 0x56, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x6f, 0x74, 0x6f, 0x22, 0x83, 0x0b, 0x0a, 0x08, 0x56, 0x6d, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x61, 0x6d, 0x64, 0x69, 0x73, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x61, 0x6d, 0x64, 0x69, @@ -515,22 +900,87 @@ var file_config_vm_proto_rawDesc = []byte{ 0x6f, 0x72, 0x64, 0x65, 0x72, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x42, 0x6f, 0x6f, 0x74, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x52, 0x09, 0x62, - 0x6f, 0x6f, 0x74, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x2a, 0x47, 0x0a, 0x06, 0x56, 0x6d, 0x4d, 0x6f, - 0x64, 0x65, 0x12, 0x06, 0x0a, 0x02, 0x50, 0x56, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x48, 0x56, - 0x4d, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x6c, 0x65, 0x72, 0x10, 0x02, 0x12, - 0x07, 0x0a, 0x03, 0x46, 0x4d, 0x4c, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x4f, 0x48, 0x59, - 0x50, 0x45, 0x52, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x45, 0x47, 0x41, 0x43, 0x59, 0x10, - 0x05, 0x2a, 0x5a, 0x0a, 0x0a, 0x56, 0x6d, 0x42, 0x6f, 0x6f, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x12, - 0x1c, 0x0a, 0x18, 0x56, 0x4d, 0x5f, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, - 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, - 0x13, 0x56, 0x4d, 0x5f, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x4c, 0x45, - 0x47, 0x41, 0x43, 0x59, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x56, 0x4d, 0x5f, 0x42, 0x4f, 0x4f, - 0x54, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x45, 0x46, 0x49, 0x10, 0x02, 0x42, 0x3d, 0x0a, - 0x15, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6c, 0x66, 0x2d, 0x65, 0x64, 0x67, 0x65, 0x2f, 0x65, 0x76, 0x65, 0x2d, 0x61, - 0x70, 0x69, 0x2f, 0x67, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x6f, 0x74, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x3f, 0x0a, 0x0a, 0x63, 0x70, 0x75, 0x5f, + 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x1c, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x70, 0x75, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x09, + 0x63, 0x70, 0x75, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x26, 0x0a, 0x0f, 0x66, 0x75, 0x6c, + 0x6c, 0x5f, 0x70, 0x63, 0x70, 0x75, 0x73, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x1d, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0d, 0x66, 0x75, 0x6c, 0x6c, 0x50, 0x63, 0x70, 0x75, 0x73, 0x4f, 0x6e, 0x6c, + 0x79, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x5f, 0x70, 0x65, 0x72, + 0x5f, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x74, 0x68, 0x72, + 0x65, 0x61, 0x64, 0x73, 0x50, 0x65, 0x72, 0x43, 0x6f, 0x72, 0x65, 0x12, 0x42, 0x0a, 0x0b, 0x6e, + 0x75, 0x6d, 0x61, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, + 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4e, 0x75, 0x6d, 0x61, 0x50, 0x6f, 0x6c, + 0x69, 0x63, 0x79, 0x52, 0x0a, 0x6e, 0x75, 0x6d, 0x61, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, + 0x45, 0x0a, 0x0c, 0x69, 0x6f, 0x5f, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, + 0x20, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x49, 0x6f, + 0x50, 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x69, 0x6f, 0x50, 0x6c, 0x61, + 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x4b, 0x0a, 0x0e, 0x69, 0x73, 0x6f, 0x6c, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x49, 0x73, 0x6f, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x69, 0x65, 0x72, 0x52, 0x0d, 0x69, 0x73, 0x6f, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, + 0x69, 0x65, 0x72, 0x12, 0x54, 0x0a, 0x11, 0x64, 0x69, 0x73, 0x72, 0x75, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x44, 0x69, 0x73, 0x72, 0x75, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x10, 0x64, 0x69, 0x73, 0x72, 0x75, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x2a, 0x47, 0x0a, 0x06, 0x56, 0x6d, 0x4d, + 0x6f, 0x64, 0x65, 0x12, 0x06, 0x0a, 0x02, 0x50, 0x56, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x48, + 0x56, 0x4d, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x6c, 0x65, 0x72, 0x10, 0x02, + 0x12, 0x07, 0x0a, 0x03, 0x46, 0x4d, 0x4c, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x4e, 0x4f, 0x48, + 0x59, 0x50, 0x45, 0x52, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x45, 0x47, 0x41, 0x43, 0x59, + 0x10, 0x05, 0x2a, 0x58, 0x0a, 0x09, 0x43, 0x70, 0x75, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, + 0x1a, 0x0a, 0x16, 0x43, 0x50, 0x55, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x43, + 0x50, 0x55, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, + 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x50, 0x55, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, + 0x5f, 0x44, 0x45, 0x44, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x9a, 0x01, 0x0a, + 0x0a, 0x4e, 0x75, 0x6d, 0x61, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x1b, 0x0a, 0x17, 0x4e, + 0x55, 0x4d, 0x41, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, + 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x4e, 0x55, 0x4d, 0x41, + 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1b, + 0x0a, 0x17, 0x4e, 0x55, 0x4d, 0x41, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x42, 0x45, + 0x53, 0x54, 0x5f, 0x45, 0x46, 0x46, 0x4f, 0x52, 0x54, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x4e, + 0x55, 0x4d, 0x41, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x52, + 0x49, 0x43, 0x54, 0x45, 0x44, 0x10, 0x03, 0x12, 0x20, 0x0a, 0x1c, 0x4e, 0x55, 0x4d, 0x41, 0x5f, + 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x53, 0x49, 0x4e, 0x47, 0x4c, 0x45, 0x5f, 0x4e, 0x55, + 0x4d, 0x41, 0x5f, 0x4e, 0x4f, 0x44, 0x45, 0x10, 0x04, 0x2a, 0x66, 0x0a, 0x0b, 0x49, 0x6f, 0x50, + 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x18, 0x49, 0x4f, 0x5f, 0x50, + 0x4c, 0x41, 0x43, 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, + 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x49, 0x4f, 0x5f, 0x50, 0x4c, 0x41, + 0x43, 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x44, 0x45, 0x44, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, + 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x49, 0x4f, 0x5f, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x4d, 0x45, + 0x4e, 0x54, 0x5f, 0x48, 0x4f, 0x55, 0x53, 0x45, 0x4b, 0x45, 0x45, 0x50, 0x49, 0x4e, 0x47, 0x10, + 0x02, 0x2a, 0x7a, 0x0a, 0x0d, 0x49, 0x73, 0x6f, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, + 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x1a, 0x49, 0x53, 0x4f, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, + 0x54, 0x49, 0x45, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x49, 0x53, 0x4f, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, + 0x54, 0x49, 0x45, 0x52, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x49, + 0x53, 0x4f, 0x4c, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x49, 0x45, 0x52, 0x5f, 0x53, 0x4f, + 0x46, 0x54, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x49, 0x53, 0x4f, 0x4c, 0x41, 0x54, 0x49, 0x4f, + 0x4e, 0x5f, 0x54, 0x49, 0x45, 0x52, 0x5f, 0x48, 0x41, 0x52, 0x44, 0x10, 0x03, 0x2a, 0x71, 0x0a, + 0x10, 0x44, 0x69, 0x73, 0x72, 0x75, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x6f, 0x6c, 0x69, 0x63, + 0x79, 0x12, 0x21, 0x0a, 0x1d, 0x44, 0x49, 0x53, 0x52, 0x55, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, + 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x44, 0x49, 0x53, 0x52, 0x55, 0x50, 0x54, 0x49, + 0x4f, 0x4e, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x41, 0x4c, 0x4c, 0x4f, 0x57, 0x10, + 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x44, 0x49, 0x53, 0x52, 0x55, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, + 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x45, 0x43, 0x54, 0x10, 0x02, + 0x2a, 0x5a, 0x0a, 0x0a, 0x56, 0x6d, 0x42, 0x6f, 0x6f, 0x74, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1c, + 0x0a, 0x18, 0x56, 0x4d, 0x5f, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, + 0x56, 0x4d, 0x5f, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x4c, 0x45, 0x47, + 0x41, 0x43, 0x59, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x56, 0x4d, 0x5f, 0x42, 0x4f, 0x4f, 0x54, + 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x45, 0x46, 0x49, 0x10, 0x02, 0x42, 0x3d, 0x0a, 0x15, + 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5a, 0x24, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x6c, 0x66, 0x2d, 0x65, 0x64, 0x67, 0x65, 0x2f, 0x65, 0x76, 0x65, 0x2d, 0x61, 0x70, + 0x69, 0x2f, 0x67, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -545,23 +995,33 @@ func file_config_vm_proto_rawDescGZIP() []byte { return file_config_vm_proto_rawDescData } -var file_config_vm_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_config_vm_proto_enumTypes = make([]protoimpl.EnumInfo, 7) var file_config_vm_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_config_vm_proto_goTypes = []interface{}{ (VmMode)(0), // 0: org.lfedge.eve.config.VmMode - (VmBootMode)(0), // 1: org.lfedge.eve.config.VmBootMode - (*VmConfig)(nil), // 2: org.lfedge.eve.config.VmConfig - (evecommon.BootOrder)(0), // 3: org.lfedge.eve.common.BootOrder + (CpuPolicy)(0), // 1: org.lfedge.eve.config.CpuPolicy + (NumaPolicy)(0), // 2: org.lfedge.eve.config.NumaPolicy + (IoPlacement)(0), // 3: org.lfedge.eve.config.IoPlacement + (IsolationTier)(0), // 4: org.lfedge.eve.config.IsolationTier + (DisruptionPolicy)(0), // 5: org.lfedge.eve.config.DisruptionPolicy + (VmBootMode)(0), // 6: org.lfedge.eve.config.VmBootMode + (*VmConfig)(nil), // 7: org.lfedge.eve.config.VmConfig + (evecommon.BootOrder)(0), // 8: org.lfedge.eve.common.BootOrder } var file_config_vm_proto_depIdxs = []int32{ 0, // 0: org.lfedge.eve.config.VmConfig.virtualizationMode:type_name -> org.lfedge.eve.config.VmMode - 1, // 1: org.lfedge.eve.config.VmConfig.boot_mode:type_name -> org.lfedge.eve.config.VmBootMode - 3, // 2: org.lfedge.eve.config.VmConfig.boot_order:type_name -> org.lfedge.eve.common.BootOrder - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 6, // 1: org.lfedge.eve.config.VmConfig.boot_mode:type_name -> org.lfedge.eve.config.VmBootMode + 8, // 2: org.lfedge.eve.config.VmConfig.boot_order:type_name -> org.lfedge.eve.common.BootOrder + 1, // 3: org.lfedge.eve.config.VmConfig.cpu_policy:type_name -> org.lfedge.eve.config.CpuPolicy + 2, // 4: org.lfedge.eve.config.VmConfig.numa_policy:type_name -> org.lfedge.eve.config.NumaPolicy + 3, // 5: org.lfedge.eve.config.VmConfig.io_placement:type_name -> org.lfedge.eve.config.IoPlacement + 4, // 6: org.lfedge.eve.config.VmConfig.isolation_tier:type_name -> org.lfedge.eve.config.IsolationTier + 5, // 7: org.lfedge.eve.config.VmConfig.disruption_policy:type_name -> org.lfedge.eve.config.DisruptionPolicy + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name } func init() { file_config_vm_proto_init() } @@ -588,7 +1048,7 @@ func file_config_vm_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_config_vm_proto_rawDesc, - NumEnums: 2, + NumEnums: 7, NumMessages: 1, NumExtensions: 0, NumServices: 0, diff --git a/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/common.pb.go b/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/common.pb.go index 477b121c27a..92054fa2820 100644 --- a/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/common.pb.go +++ b/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/common.pb.go @@ -177,6 +177,12 @@ type ErrorInfo struct { Severity Severity `protobuf:"varint,3,opt,name=severity,proto3,enum=org.lfedge.eve.info.Severity" json:"severity,omitempty"` // Severity level of the error. Entities []*DeviceEntity `protobuf:"bytes,4,rep,name=entities,proto3" json:"entities,omitempty"` // Device entities referenced in the description or retry condition. RetryCondition string `protobuf:"bytes,5,opt,name=retry_condition,json=retryCondition,proto3" json:"retry_condition,omitempty"` // Condition under which the operation may be retried. + // Machine-parseable, namespaced error token (e.g. + // "cpu.placement.insufficient"), stable across EVE releases, so the + // controller can react programmatically without parsing the free-text + // description. Namespaces form an open registry extended additively + // per domain; an empty value means no structured code is available. + ErrorCode string `protobuf:"bytes,6,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"` } func (x *ErrorInfo) Reset() { @@ -246,6 +252,13 @@ func (x *ErrorInfo) GetRetryCondition() string { return "" } +func (x *ErrorInfo) GetErrorCode() string { + if x != nil { + return x.ErrorCode + } + return "" +} + // DeviceEntity contains the device entity details type DeviceEntity struct { state protoimpl.MessageState @@ -317,7 +330,7 @@ var file_info_common_proto_rawDesc = []byte{ 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8a, 0x02, 0x0a, 0x09, 0x45, 0x72, + 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xa9, 0x02, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, @@ -334,43 +347,45 @@ var file_info_common_proto_rawDesc = []byte{ 0x69, 0x74, 0x79, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x74, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65, 0x74, 0x72, 0x79, 0x43, 0x6f, 0x6e, - 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x81, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, - 0x65, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x33, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, - 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x1b, 0x0a, 0x09, - 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x6e, 0x74, - 0x69, 0x74, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x2a, 0x63, 0x0a, 0x08, 0x53, 0x65, - 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x45, 0x56, 0x45, 0x52, 0x49, - 0x54, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x45, 0x56, 0x45, 0x52, 0x49, 0x54, 0x59, 0x5f, 0x4e, 0x4f, 0x54, - 0x49, 0x43, 0x45, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x45, 0x56, 0x45, 0x52, 0x49, 0x54, - 0x59, 0x5f, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, - 0x45, 0x56, 0x45, 0x52, 0x49, 0x54, 0x59, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x2a, - 0x99, 0x02, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x4e, - 0x54, 0x49, 0x54, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x42, 0x41, 0x53, - 0x45, 0x5f, 0x4f, 0x53, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, - 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x41, 0x44, 0x41, 0x50, 0x54, 0x45, 0x52, 0x10, - 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x56, 0x41, 0x55, 0x4c, - 0x54, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x54, - 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, 0x45, - 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4e, - 0x43, 0x45, 0x10, 0x05, 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x50, - 0x4f, 0x52, 0x54, 0x10, 0x06, 0x12, 0x12, 0x0a, 0x0e, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, - 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x07, 0x12, 0x1b, 0x0a, 0x17, 0x45, 0x4e, 0x54, - 0x49, 0x54, 0x59, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x49, 0x4e, 0x53, 0x54, - 0x41, 0x4e, 0x43, 0x45, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, - 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x52, 0x45, 0x45, 0x10, 0x09, 0x12, - 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x45, 0x4e, - 0x54, 0x5f, 0x42, 0x4c, 0x4f, 0x42, 0x10, 0x0a, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x4e, 0x54, 0x49, - 0x54, 0x59, 0x5f, 0x56, 0x4f, 0x4c, 0x55, 0x4d, 0x45, 0x10, 0x0b, 0x42, 0x39, 0x0a, 0x13, 0x6f, - 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, - 0x66, 0x6f, 0x5a, 0x22, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, - 0x66, 0x2d, 0x65, 0x64, 0x67, 0x65, 0x2f, 0x65, 0x76, 0x65, 0x2d, 0x61, 0x70, 0x69, 0x2f, 0x67, - 0x6f, 0x2f, 0x69, 0x6e, 0x66, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, + 0x63, 0x6f, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x43, 0x6f, 0x64, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, + 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x33, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x6e, 0x74, 0x69, + 0x74, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x2a, 0x63, 0x0a, 0x08, 0x53, 0x65, 0x76, + 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x45, 0x56, 0x45, 0x52, 0x49, 0x54, + 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, + 0x13, 0x0a, 0x0f, 0x53, 0x45, 0x56, 0x45, 0x52, 0x49, 0x54, 0x59, 0x5f, 0x4e, 0x4f, 0x54, 0x49, + 0x43, 0x45, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x45, 0x56, 0x45, 0x52, 0x49, 0x54, 0x59, + 0x5f, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x45, + 0x56, 0x45, 0x52, 0x49, 0x54, 0x59, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x2a, 0x99, + 0x02, 0x0a, 0x06, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x4e, 0x54, + 0x49, 0x54, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x42, 0x41, 0x53, 0x45, + 0x5f, 0x4f, 0x53, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, + 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x5f, 0x41, 0x44, 0x41, 0x50, 0x54, 0x45, 0x52, 0x10, 0x02, + 0x12, 0x10, 0x0a, 0x0c, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x56, 0x41, 0x55, 0x4c, 0x54, + 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x54, 0x54, + 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x4e, + 0x54, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4e, 0x43, + 0x45, 0x10, 0x05, 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x50, 0x4f, + 0x52, 0x54, 0x10, 0x06, 0x12, 0x12, 0x0a, 0x0e, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x4e, + 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x07, 0x12, 0x1b, 0x0a, 0x17, 0x45, 0x4e, 0x54, 0x49, + 0x54, 0x59, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, + 0x4e, 0x43, 0x45, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, + 0x43, 0x4f, 0x4e, 0x54, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x52, 0x45, 0x45, 0x10, 0x09, 0x12, 0x17, + 0x0a, 0x13, 0x45, 0x4e, 0x54, 0x49, 0x54, 0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x45, 0x4e, 0x54, + 0x5f, 0x42, 0x4c, 0x4f, 0x42, 0x10, 0x0a, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x4e, 0x54, 0x49, 0x54, + 0x59, 0x5f, 0x56, 0x4f, 0x4c, 0x55, 0x4d, 0x45, 0x10, 0x0b, 0x42, 0x39, 0x0a, 0x13, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x5a, 0x22, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x66, + 0x2d, 0x65, 0x64, 0x67, 0x65, 0x2f, 0x65, 0x76, 0x65, 0x2d, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x6f, + 0x2f, 0x69, 0x6e, 0x66, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/hardware.pb.go b/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/hardware.pb.go index 8129b65bbb6..2ca59c2ca9b 100644 --- a/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/hardware.pb.go +++ b/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/hardware.pb.go @@ -81,6 +81,117 @@ func (NetworkDeviceType) EnumDescriptor() ([]byte, []int) { return file_info_hardware_proto_rawDescGZIP(), []int{0} } +// CoreClass distinguishes core types on heterogeneous (hybrid) CPUs, +// e.g. Intel P/E cores or ARM big.LITTLE. Homogeneous parts report +// CORE_CLASS_UNSPECIFIED for all cores. +type CoreClass int32 + +const ( + CoreClass_CORE_CLASS_UNSPECIFIED CoreClass = 0 + CoreClass_CORE_CLASS_PERFORMANCE CoreClass = 1 // e.g. Intel P-core, ARM "big" + CoreClass_CORE_CLASS_EFFICIENCY CoreClass = 2 // e.g. Intel E-core, ARM "LITTLE" + CoreClass_CORE_CLASS_LOW_POWER CoreClass = 3 // e.g. Intel LP E-core +) + +// Enum value maps for CoreClass. +var ( + CoreClass_name = map[int32]string{ + 0: "CORE_CLASS_UNSPECIFIED", + 1: "CORE_CLASS_PERFORMANCE", + 2: "CORE_CLASS_EFFICIENCY", + 3: "CORE_CLASS_LOW_POWER", + } + CoreClass_value = map[string]int32{ + "CORE_CLASS_UNSPECIFIED": 0, + "CORE_CLASS_PERFORMANCE": 1, + "CORE_CLASS_EFFICIENCY": 2, + "CORE_CLASS_LOW_POWER": 3, + } +) + +func (x CoreClass) Enum() *CoreClass { + p := new(CoreClass) + *p = x + return p +} + +func (x CoreClass) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CoreClass) Descriptor() protoreflect.EnumDescriptor { + return file_info_hardware_proto_enumTypes[1].Descriptor() +} + +func (CoreClass) Type() protoreflect.EnumType { + return &file_info_hardware_proto_enumTypes[1] +} + +func (x CoreClass) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CoreClass.Descriptor instead. +func (CoreClass) EnumDescriptor() ([]byte, []int) { + return file_info_hardware_proto_rawDescGZIP(), []int{1} +} + +// CacheLevel identifies a CPU cache level for CacheDomain. +type CacheLevel int32 + +const ( + CacheLevel_CACHE_LEVEL_UNSPECIFIED CacheLevel = 0 + CacheLevel_CACHE_LEVEL_L1D CacheLevel = 1 // level-1 data cache + CacheLevel_CACHE_LEVEL_L1I CacheLevel = 2 // level-1 instruction cache + CacheLevel_CACHE_LEVEL_L2 CacheLevel = 3 + CacheLevel_CACHE_LEVEL_L3 CacheLevel = 4 +) + +// Enum value maps for CacheLevel. +var ( + CacheLevel_name = map[int32]string{ + 0: "CACHE_LEVEL_UNSPECIFIED", + 1: "CACHE_LEVEL_L1D", + 2: "CACHE_LEVEL_L1I", + 3: "CACHE_LEVEL_L2", + 4: "CACHE_LEVEL_L3", + } + CacheLevel_value = map[string]int32{ + "CACHE_LEVEL_UNSPECIFIED": 0, + "CACHE_LEVEL_L1D": 1, + "CACHE_LEVEL_L1I": 2, + "CACHE_LEVEL_L2": 3, + "CACHE_LEVEL_L3": 4, + } +) + +func (x CacheLevel) Enum() *CacheLevel { + p := new(CacheLevel) + *p = x + return p +} + +func (x CacheLevel) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CacheLevel) Descriptor() protoreflect.EnumDescriptor { + return file_info_hardware_proto_enumTypes[2].Descriptor() +} + +func (CacheLevel) Type() protoreflect.EnumType { + return &file_info_hardware_proto_enumTypes[2] +} + +func (x CacheLevel) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CacheLevel.Descriptor instead. +func (CacheLevel) EnumDescriptor() ([]byte, []int) { + return file_info_hardware_proto_rawDescGZIP(), []int{2} +} + type USBAddress struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -388,6 +499,15 @@ type PCIDevice struct { SubclassId uint64 `protobuf:"varint,9,opt,name=subclass_id,json=subclassId,proto3" json:"subclass_id,omitempty"` // IOMMU group names apparently are strings: https://elixir.bootlin.com/linux/v6.16.9/source/drivers/iommu/iommu.c#L1101 IommuGroup string `protobuf:"bytes,10,opt,name=iommu_group,json=iommuGroup,proto3" json:"iommu_group,omitempty"` + // NUMA node this device's DMA/interrupts are local to, from sysfs + // /sys/bus/pci/devices//numa_node. -1 means the platform does + // not report an affinity. Unset means EVE did not populate it. + // Cross-references CPU.numa_node in CPUInfo, enabling NUMA-local + // placement of a workload relative to its passthrough device. + NumaNode *int32 `protobuf:"varint,11,opt,name=numa_node,json=numaNode,proto3,oneof" json:"numa_node,omitempty"` + // Logical CPU ids local to this device, from sysfs local_cpulist. + // Cross-references CPU.id in CPUInfo. + LocalCpuIds []uint32 `protobuf:"varint,12,rep,packed,name=local_cpu_ids,json=localCpuIds,proto3" json:"local_cpu_ids,omitempty"` } func (x *PCIDevice) Reset() { @@ -492,6 +612,20 @@ func (x *PCIDevice) GetIommuGroup() string { return "" } +func (x *PCIDevice) GetNumaNode() int32 { + if x != nil && x.NumaNode != nil { + return *x.NumaNode + } + return 0 +} + +func (x *PCIDevice) GetLocalCpuIds() []uint32 { + if x != nil { + return x.LocalCpuIds + } + return nil +} + type SerialPort struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -762,15 +896,28 @@ func (x *BIOS) GetAttributes() map[string]string { return nil } +// One entry per logical CPU (SMT thread) — the schedulable unit. +// The topology coordinates (socket/core/NUMA/cache ids) are opaque +// grouping keys: equal values mean "same domain"; the values are not +// guaranteed contiguous or zero-based. Logical CPUs sharing +// (socket_id, core_id) are SMT siblings of one physical core. type CPU struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Model string `protobuf:"bytes,1,opt,name=model,proto3" json:"model,omitempty"` - Vendor string `protobuf:"bytes,2,opt,name=vendor,proto3" json:"vendor,omitempty"` // e.g. "GenuineIntel" - Id uint32 `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"` // logical CPU id - Freq uint64 `protobuf:"varint,4,opt,name=freq,proto3" json:"freq,omitempty"` // nominal frequency + Model string `protobuf:"bytes,1,opt,name=model,proto3" json:"model,omitempty"` + Vendor string `protobuf:"bytes,2,opt,name=vendor,proto3" json:"vendor,omitempty"` // e.g. "GenuineIntel" + Id uint32 `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"` // logical CPU id + Freq uint64 `protobuf:"varint,4,opt,name=freq,proto3" json:"freq,omitempty"` // nominal frequency + SocketId uint32 `protobuf:"varint,5,opt,name=socket_id,json=socketId,proto3" json:"socket_id,omitempty"` // physical package (socket) + CoreId uint32 `protobuf:"varint,6,opt,name=core_id,json=coreId,proto3" json:"core_id,omitempty"` // physical core within the socket + NumaNode uint32 `protobuf:"varint,7,opt,name=numa_node,json=numaNode,proto3" json:"numa_node,omitempty"` // NUMA node the CPU belongs to + L2Id uint32 `protobuf:"varint,8,opt,name=l2_id,json=l2Id,proto3" json:"l2_id,omitempty"` // L2 cache domain id + L3Id uint32 `protobuf:"varint,9,opt,name=l3_id,json=l3Id,proto3" json:"l3_id,omitempty"` // L3 cache domain id + CoreClass CoreClass `protobuf:"varint,10,opt,name=core_class,json=coreClass,proto3,enum=org.lfedge.eve.info.CoreClass" json:"core_class,omitempty"` // P/E/LP class on hybrid parts + BaseFreqKhz uint64 `protobuf:"varint,11,opt,name=base_freq_khz,json=baseFreqKhz,proto3" json:"base_freq_khz,omitempty"` // base frequency, kHz + MaxFreqKhz uint64 `protobuf:"varint,12,opt,name=max_freq_khz,json=maxFreqKhz,proto3" json:"max_freq_khz,omitempty"` // max (turbo) frequency, kHz } func (x *CPU) Reset() { @@ -833,18 +980,223 @@ func (x *CPU) GetFreq() uint64 { return 0 } +func (x *CPU) GetSocketId() uint32 { + if x != nil { + return x.SocketId + } + return 0 +} + +func (x *CPU) GetCoreId() uint32 { + if x != nil { + return x.CoreId + } + return 0 +} + +func (x *CPU) GetNumaNode() uint32 { + if x != nil { + return x.NumaNode + } + return 0 +} + +func (x *CPU) GetL2Id() uint32 { + if x != nil { + return x.L2Id + } + return 0 +} + +func (x *CPU) GetL3Id() uint32 { + if x != nil { + return x.L3Id + } + return 0 +} + +func (x *CPU) GetCoreClass() CoreClass { + if x != nil { + return x.CoreClass + } + return CoreClass_CORE_CLASS_UNSPECIFIED +} + +func (x *CPU) GetBaseFreqKhz() uint64 { + if x != nil { + return x.BaseFreqKhz + } + return 0 +} + +func (x *CPU) GetMaxFreqKhz() uint64 { + if x != nil { + return x.MaxFreqKhz + } + return 0 +} + +// CacheDomain describes one cache instance and the logical CPUs sharing +// it — the cache→cores linkage (values of CPU.id). Note that on some +// parts a cache domain spans multiple physical cores that are NOT SMT +// siblings (e.g. an Intel E-core module shares one L2 across four +// distinct cores); sibling grouping must use (socket_id, core_id), never +// a cache id. +type CacheDomain struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Level CacheLevel `protobuf:"varint,1,opt,name=level,proto3,enum=org.lfedge.eve.info.CacheLevel" json:"level,omitempty"` + SizeBytes uint64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + // Logical CPU ids (CPU.id) sharing this cache instance. + CpuIds []uint32 `protobuf:"varint,3,rep,packed,name=cpu_ids,json=cpuIds,proto3" json:"cpu_ids,omitempty"` + // Domain id as reported by the platform; opaque grouping key, + // matching CPU.l2_id / CPU.l3_id for the respective level. + Id uint32 `protobuf:"varint,4,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *CacheDomain) Reset() { + *x = CacheDomain{} + if protoimpl.UnsafeEnabled { + mi := &file_info_hardware_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CacheDomain) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CacheDomain) ProtoMessage() {} + +func (x *CacheDomain) ProtoReflect() protoreflect.Message { + mi := &file_info_hardware_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CacheDomain.ProtoReflect.Descriptor instead. +func (*CacheDomain) Descriptor() ([]byte, []int) { + return file_info_hardware_proto_rawDescGZIP(), []int{10} +} + +func (x *CacheDomain) GetLevel() CacheLevel { + if x != nil { + return x.Level + } + return CacheLevel_CACHE_LEVEL_UNSPECIFIED +} + +func (x *CacheDomain) GetSizeBytes() uint64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +func (x *CacheDomain) GetCpuIds() []uint32 { + if x != nil { + return x.CpuIds + } + return nil +} + +func (x *CacheDomain) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +// CPUCapabilities reports CPU-hardware (silicon) resource-control +// features. Kernel/boot-level facts are NOT CPU properties and are +// reported separately in NodeCapabilities. +type CPUCapabilities struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + RdtL3Cat bool `protobuf:"varint,1,opt,name=rdt_l3_cat,json=rdtL3Cat,proto3" json:"rdt_l3_cat,omitempty"` // Intel RDT L3 cache allocation available + RdtMba bool `protobuf:"varint,2,opt,name=rdt_mba,json=rdtMba,proto3" json:"rdt_mba,omitempty"` // Intel RDT memory-bandwidth allocation available + NumClos uint32 `protobuf:"varint,3,opt,name=num_clos,json=numClos,proto3" json:"num_clos,omitempty"` // number of RDT classes of service (if rdt_*) +} + +func (x *CPUCapabilities) Reset() { + *x = CPUCapabilities{} + if protoimpl.UnsafeEnabled { + mi := &file_info_hardware_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CPUCapabilities) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CPUCapabilities) ProtoMessage() {} + +func (x *CPUCapabilities) ProtoReflect() protoreflect.Message { + mi := &file_info_hardware_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CPUCapabilities.ProtoReflect.Descriptor instead. +func (*CPUCapabilities) Descriptor() ([]byte, []int) { + return file_info_hardware_proto_rawDescGZIP(), []int{11} +} + +func (x *CPUCapabilities) GetRdtL3Cat() bool { + if x != nil { + return x.RdtL3Cat + } + return false +} + +func (x *CPUCapabilities) GetRdtMba() bool { + if x != nil { + return x.RdtMba + } + return false +} + +func (x *CPUCapabilities) GetNumClos() uint32 { + if x != nil { + return x.NumClos + } + return 0 +} + type CPUInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields Cpus []*CPU `protobuf:"bytes,1,rep,name=cpus,proto3" json:"cpus,omitempty"` + // Cache domains with their cache→cores linkage. + Caches []*CacheDomain `protobuf:"bytes,2,rep,name=caches,proto3" json:"caches,omitempty"` + Capabilities *CPUCapabilities `protobuf:"bytes,3,opt,name=capabilities,proto3" json:"capabilities,omitempty"` } func (x *CPUInfo) Reset() { *x = CPUInfo{} if protoimpl.UnsafeEnabled { - mi := &file_info_hardware_proto_msgTypes[10] + mi := &file_info_hardware_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -857,7 +1209,7 @@ func (x *CPUInfo) String() string { func (*CPUInfo) ProtoMessage() {} func (x *CPUInfo) ProtoReflect() protoreflect.Message { - mi := &file_info_hardware_proto_msgTypes[10] + mi := &file_info_hardware_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -870,7 +1222,7 @@ func (x *CPUInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use CPUInfo.ProtoReflect.Descriptor instead. func (*CPUInfo) Descriptor() ([]byte, []int) { - return file_info_hardware_proto_rawDescGZIP(), []int{10} + return file_info_hardware_proto_rawDescGZIP(), []int{12} } func (x *CPUInfo) GetCpus() []*CPU { @@ -880,6 +1232,101 @@ func (x *CPUInfo) GetCpus() []*CPU { return nil } +func (x *CPUInfo) GetCaches() []*CacheDomain { + if x != nil { + return x.Caches + } + return nil +} + +func (x *CPUInfo) GetCapabilities() *CPUCapabilities { + if x != nil { + return x.Capabilities + } + return nil +} + +// NodeCapabilities reports granular kernel/boot-level facts relevant to +// workload CPU isolation. Deliberately low-level and factual: higher-level +// notions (e.g. which isolation tier is achievable for a given workload +// type) are derived by the consumer from these ingredients rather than +// precomputed by the node, so new ingredients can be added additively +// without changing the meaning of existing ones. +// +// Scope: this message reports only what the HARDWARE and the RUNNING KERNEL +// provide. What the EVE software itself is able to do with those facts is a +// separate concern and is reported through the device's software-capability +// channels (ZInfoDevice.optional_capabilities and ZInfoDevice.api_capability), +// so a hardware fact never has to change meaning when EVE gains or loses a +// feature. A consumer deciding whether a feature is offerable must consult +// both: hardware/kernel capable AND EVE software capable. +type NodeCapabilities struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Logical CPUs currently isolated from the scheduler by the + // running kernel (isolcpus), from /sys/devices/system/cpu/isolated. + IsolatedCpuIds []uint32 `protobuf:"varint,1,rep,packed,name=isolated_cpu_ids,json=isolatedCpuIds,proto3" json:"isolated_cpu_ids,omitempty"` + // Logical CPUs with the scheduler tick shed (nohz_full). + NohzFullCpuIds []uint32 `protobuf:"varint,2,rep,packed,name=nohz_full_cpu_ids,json=nohzFullCpuIds,proto3" json:"nohz_full_cpu_ids,omitempty"` + // Logical CPUs whose RCU callbacks are offloaded (rcu_nocbs). + RcuNocbsCpuIds []uint32 `protobuf:"varint,3,rep,packed,name=rcu_nocbs_cpu_ids,json=rcuNocbsCpuIds,proto3" json:"rcu_nocbs_cpu_ids,omitempty"` +} + +func (x *NodeCapabilities) Reset() { + *x = NodeCapabilities{} + if protoimpl.UnsafeEnabled { + mi := &file_info_hardware_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeCapabilities) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeCapabilities) ProtoMessage() {} + +func (x *NodeCapabilities) ProtoReflect() protoreflect.Message { + mi := &file_info_hardware_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeCapabilities.ProtoReflect.Descriptor instead. +func (*NodeCapabilities) Descriptor() ([]byte, []int) { + return file_info_hardware_proto_rawDescGZIP(), []int{13} +} + +func (x *NodeCapabilities) GetIsolatedCpuIds() []uint32 { + if x != nil { + return x.IsolatedCpuIds + } + return nil +} + +func (x *NodeCapabilities) GetNohzFullCpuIds() []uint32 { + if x != nil { + return x.NohzFullCpuIds + } + return nil +} + +func (x *NodeCapabilities) GetRcuNocbsCpuIds() []uint32 { + if x != nil { + return x.RcuNocbsCpuIds + } + return nil +} + type TPM struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -894,7 +1341,7 @@ type TPM struct { func (x *TPM) Reset() { *x = TPM{} if protoimpl.UnsafeEnabled { - mi := &file_info_hardware_proto_msgTypes[11] + mi := &file_info_hardware_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -907,7 +1354,7 @@ func (x *TPM) String() string { func (*TPM) ProtoMessage() {} func (x *TPM) ProtoReflect() protoreflect.Message { - mi := &file_info_hardware_proto_msgTypes[11] + mi := &file_info_hardware_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -920,7 +1367,7 @@ func (x *TPM) ProtoReflect() protoreflect.Message { // Deprecated: Use TPM.ProtoReflect.Descriptor instead. func (*TPM) Descriptor() ([]byte, []int) { - return file_info_hardware_proto_rawDescGZIP(), []int{11} + return file_info_hardware_proto_rawDescGZIP(), []int{14} } func (x *TPM) GetPresent() bool { @@ -978,6 +1425,8 @@ type HardwareInventory struct { WatchdogPresent bool `protobuf:"varint,13,opt,name=watchdog_present,json=watchdogPresent,proto3" json:"watchdog_present,omitempty"` Tpm *TPM `protobuf:"bytes,14,opt,name=tpm,proto3" json:"tpm,omitempty"` StatusLedPresent bool `protobuf:"varint,15,opt,name=status_led_present,json=statusLedPresent,proto3" json:"status_led_present,omitempty"` + // Granular kernel/boot-level isolation facts (node, not CPU, properties). + NodeCapabilities *NodeCapabilities `protobuf:"bytes,16,opt,name=node_capabilities,json=nodeCapabilities,proto3" json:"node_capabilities,omitempty"` // Free-form extension for things not modeled yet. // New keys must be promoted to typed fields in a follow-up. Misc map[string]string `protobuf:"bytes,1000,rep,name=misc,proto3" json:"misc,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -986,7 +1435,7 @@ type HardwareInventory struct { func (x *HardwareInventory) Reset() { *x = HardwareInventory{} if protoimpl.UnsafeEnabled { - mi := &file_info_hardware_proto_msgTypes[12] + mi := &file_info_hardware_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -999,7 +1448,7 @@ func (x *HardwareInventory) String() string { func (*HardwareInventory) ProtoMessage() {} func (x *HardwareInventory) ProtoReflect() protoreflect.Message { - mi := &file_info_hardware_proto_msgTypes[12] + mi := &file_info_hardware_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1012,7 +1461,7 @@ func (x *HardwareInventory) ProtoReflect() protoreflect.Message { // Deprecated: Use HardwareInventory.ProtoReflect.Descriptor instead. func (*HardwareInventory) Descriptor() ([]byte, []int) { - return file_info_hardware_proto_rawDescGZIP(), []int{12} + return file_info_hardware_proto_rawDescGZIP(), []int{15} } func (x *HardwareInventory) GetPciDevices() []*PCIDevice { @@ -1099,6 +1548,13 @@ func (x *HardwareInventory) GetStatusLedPresent() bool { return false } +func (x *HardwareInventory) GetNodeCapabilities() *NodeCapabilities { + if x != nil { + return x.NodeCapabilities + } + return nil +} + func (x *HardwareInventory) GetMisc() map[string]string { if x != nil { return x.Misc @@ -1152,7 +1608,7 @@ var file_info_hardware_proto_rawDesc = []byte{ 0x03, 0x62, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, - 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xb3, 0x03, 0x0a, 0x09, 0x50, 0x43, 0x49, + 0x66, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x87, 0x04, 0x0a, 0x09, 0x50, 0x43, 0x49, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5f, 0x0a, 0x19, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x63, 0x69, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, @@ -1178,136 +1634,211 @@ var file_info_hardware_proto_rawDesc = []byte{ 0x01, 0x28, 0x04, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6f, 0x6d, 0x6d, 0x75, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6f, 0x6d, 0x6d, 0x75, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x42, 0x1c, 0x0a, 0x1a, 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x63, 0x69, 0x5f, - 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xa3, - 0x01, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x3b, 0x0a, - 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x75, 0x73, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, - 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6f, - 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x69, 0x6f, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x10, 0x0a, - 0x03, 0x69, 0x72, 0x71, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x69, 0x72, 0x71, 0x12, - 0x18, 0x0a, 0x07, 0x64, 0x65, 0x76, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x64, 0x65, 0x76, 0x70, 0x61, 0x74, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x22, 0x6b, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, 0x65, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x12, 0x20, 0x0a, 0x09, 0x6e, 0x75, 0x6d, 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, 0x08, 0x6e, 0x75, 0x6d, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x88, + 0x01, 0x01, 0x12, 0x22, 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x63, 0x70, 0x75, 0x5f, + 0x69, 0x64, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0b, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x43, 0x70, 0x75, 0x49, 0x64, 0x73, 0x42, 0x1c, 0x0a, 0x1a, 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x5f, 0x70, 0x63, 0x69, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x6e, 0x75, 0x6d, 0x61, 0x5f, 0x6e, 0x6f, + 0x64, 0x65, 0x22, 0xa3, 0x01, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x50, 0x6f, 0x72, + 0x74, 0x12, 0x3b, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x75, 0x73, 0x50, 0x61, 0x72, 0x65, 0x6e, - 0x74, 0x48, 0x00, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x16, - 0x0a, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, - 0x74, 0x22, 0xeb, 0x01, 0x0a, 0x0d, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x44, 0x65, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x75, 0x73, 0x50, 0x61, 0x72, - 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, - 0x12, 0x16, 0x0a, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, - 0x74, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x63, 0x5f, 0x61, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x61, 0x63, 0x41, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x70, 0x65, 0x65, 0x64, 0x5f, 0x6d, - 0x62, 0x70, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x73, 0x70, 0x65, 0x65, 0x64, - 0x4d, 0x62, 0x70, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x22, - 0xc3, 0x01, 0x0a, 0x04, 0x42, 0x49, 0x4f, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x6e, 0x64, - 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, - 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4a, 0x0a, 0x0a, 0x61, 0x74, - 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0xe8, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, - 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x49, 0x4f, 0x53, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, - 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, - 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, - 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57, 0x0a, 0x03, 0x43, 0x50, 0x55, 0x12, 0x14, 0x0a, 0x05, - 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x64, - 0x65, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, - 0x65, 0x71, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x66, 0x72, 0x65, 0x71, 0x22, 0x37, - 0x0a, 0x07, 0x43, 0x50, 0x55, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x2c, 0x0a, 0x04, 0x63, 0x70, 0x75, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, - 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x50, - 0x55, 0x52, 0x04, 0x63, 0x70, 0x75, 0x73, 0x22, 0x91, 0x01, 0x0a, 0x03, 0x54, 0x50, 0x4d, 0x12, - 0x18, 0x0a, 0x07, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x6d, 0x61, 0x6e, - 0x75, 0x66, 0x61, 0x63, 0x74, 0x75, 0x72, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x6d, 0x61, 0x6e, 0x75, 0x66, 0x61, 0x63, 0x74, 0x75, 0x72, 0x65, 0x72, 0x12, 0x29, 0x0a, - 0x10, 0x66, 0x69, 0x72, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x66, 0x69, 0x72, 0x6d, 0x77, 0x61, 0x72, - 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x70, 0x65, 0x63, - 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x73, 0x70, 0x65, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xbc, 0x06, 0x0a, 0x11, - 0x48, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x72, - 0x79, 0x12, 0x3f, 0x0a, 0x0b, 0x70, 0x63, 0x69, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x50, 0x43, 0x49, - 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x52, 0x0a, 0x70, 0x63, 0x69, 0x44, 0x65, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x12, 0x3f, 0x0a, 0x0b, 0x75, 0x73, 0x62, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, - 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x55, 0x53, - 0x42, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x52, 0x0a, 0x75, 0x73, 0x62, 0x44, 0x65, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x12, 0x46, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x5f, 0x64, 0x65, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x72, - 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, - 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x50, 0x6f, 0x72, 0x74, 0x52, 0x0d, 0x73, 0x65, - 0x72, 0x69, 0x61, 0x6c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x4b, 0x0a, 0x0f, 0x6e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x06, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, - 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x3f, 0x0a, 0x0b, 0x63, 0x61, 0x6e, 0x5f, - 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x41, 0x4e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x52, 0x0a, 0x63, - 0x61, 0x6e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x04, 0x62, 0x69, 0x6f, - 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, - 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x49, - 0x4f, 0x53, 0x52, 0x04, 0x62, 0x69, 0x6f, 0x73, 0x12, 0x37, 0x0a, 0x08, 0x63, 0x70, 0x75, 0x5f, - 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x67, + 0x74, 0x48, 0x00, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x21, + 0x0a, 0x0c, 0x69, 0x6f, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x6f, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x61, 0x6e, 0x67, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x72, 0x71, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, + 0x69, 0x72, 0x71, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x76, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x65, 0x76, 0x70, 0x61, 0x74, 0x68, 0x42, 0x09, 0x0a, + 0x07, 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x22, 0x6b, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, + 0x65, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x75, 0x73, 0x50, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x88, + 0x01, 0x01, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x22, 0xeb, 0x01, 0x0a, 0x0d, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, + 0x6b, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, + 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x75, + 0x73, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x88, 0x01, 0x01, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, - 0x2e, 0x43, 0x50, 0x55, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x63, 0x70, 0x75, 0x49, 0x6e, 0x66, - 0x6f, 0x12, 0x2c, 0x0a, 0x12, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, - 0x79, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, - 0x2e, 0x0a, 0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x11, 0x74, 0x6f, - 0x74, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, - 0x29, 0x0a, 0x10, 0x77, 0x61, 0x74, 0x63, 0x68, 0x64, 0x6f, 0x67, 0x5f, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x77, 0x61, 0x74, 0x63, 0x68, - 0x64, 0x6f, 0x67, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x2a, 0x0a, 0x03, 0x74, 0x70, - 0x6d, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, - 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x54, 0x50, - 0x4d, 0x52, 0x03, 0x74, 0x70, 0x6d, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x5f, 0x6c, 0x65, 0x64, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x0f, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x10, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4c, 0x65, 0x64, 0x50, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x74, 0x12, 0x45, 0x0a, 0x04, 0x6d, 0x69, 0x73, 0x63, 0x18, 0xe8, 0x07, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x48, 0x61, 0x72, 0x64, 0x77, 0x61, - 0x72, 0x65, 0x49, 0x6e, 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x72, 0x79, 0x2e, 0x4d, 0x69, 0x73, 0x63, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x6d, 0x69, 0x73, 0x63, 0x1a, 0x37, 0x0a, 0x09, 0x4d, - 0x69, 0x73, 0x63, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, 0x0c, 0x10, 0x0d, 0x2a, 0xd2, 0x01, 0x0a, 0x11, 0x4e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x1f, 0x0a, 0x1b, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, 0x49, - 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, - 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, - 0x49, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x54, 0x48, 0x45, 0x52, 0x4e, 0x45, - 0x54, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, - 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x49, 0x46, 0x49, 0x10, - 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, - 0x49, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x57, 0x41, 0x4e, 0x10, 0x06, 0x12, - 0x1e, 0x0a, 0x1a, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, 0x49, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x54, 0x48, 0x5f, 0x50, 0x46, 0x10, 0x0b, 0x12, - 0x1e, 0x0a, 0x1a, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, 0x49, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x54, 0x48, 0x5f, 0x56, 0x46, 0x10, 0x0c, 0x42, + 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x63, 0x5f, + 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6d, + 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x70, 0x65, + 0x65, 0x64, 0x5f, 0x6d, 0x62, 0x70, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x73, + 0x70, 0x65, 0x65, 0x64, 0x4d, 0x62, 0x70, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x22, 0xc3, 0x01, 0x0a, 0x04, 0x42, 0x49, 0x4f, 0x53, 0x12, 0x16, 0x0a, 0x06, + 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, + 0x6e, 0x64, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4a, + 0x0a, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0xe8, 0x07, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x49, 0x4f, 0x53, 0x2e, 0x41, + 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, + 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x41, 0x74, + 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xd9, 0x02, 0x0a, 0x03, 0x43, 0x50, + 0x55, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x6e, 0x64, 0x6f, + 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x71, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x66, + 0x72, 0x65, 0x71, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x69, 0x64, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x64, + 0x12, 0x17, 0x0a, 0x07, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x06, 0x63, 0x6f, 0x72, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x75, 0x6d, + 0x61, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x6e, 0x75, + 0x6d, 0x61, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x13, 0x0a, 0x05, 0x6c, 0x32, 0x5f, 0x69, 0x64, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6c, 0x32, 0x49, 0x64, 0x12, 0x13, 0x0a, 0x05, 0x6c, + 0x33, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6c, 0x33, 0x49, 0x64, + 0x12, 0x3d, 0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0x43, + 0x6c, 0x61, 0x73, 0x73, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x12, + 0x22, 0x0a, 0x0d, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x66, 0x72, 0x65, 0x71, 0x5f, 0x6b, 0x68, 0x7a, + 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x62, 0x61, 0x73, 0x65, 0x46, 0x72, 0x65, 0x71, + 0x4b, 0x68, 0x7a, 0x12, 0x20, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x72, 0x65, 0x71, 0x5f, + 0x6b, 0x68, 0x7a, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x46, 0x72, + 0x65, 0x71, 0x4b, 0x68, 0x7a, 0x22, 0x8c, 0x01, 0x0a, 0x0b, 0x43, 0x61, 0x63, 0x68, 0x65, 0x44, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x35, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, + 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x09, 0x73, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x63, + 0x70, 0x75, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x06, 0x63, 0x70, + 0x75, 0x49, 0x64, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x02, 0x69, 0x64, 0x22, 0x63, 0x0a, 0x0f, 0x43, 0x50, 0x55, 0x43, 0x61, 0x70, 0x61, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x0a, 0x72, 0x64, 0x74, 0x5f, 0x6c, + 0x33, 0x5f, 0x63, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x64, 0x74, + 0x4c, 0x33, 0x43, 0x61, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x72, 0x64, 0x74, 0x5f, 0x6d, 0x62, 0x61, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x72, 0x64, 0x74, 0x4d, 0x62, 0x61, 0x12, 0x19, + 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x6c, 0x6f, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x07, 0x6e, 0x75, 0x6d, 0x43, 0x6c, 0x6f, 0x73, 0x22, 0xbb, 0x01, 0x0a, 0x07, 0x43, 0x50, + 0x55, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x2c, 0x0a, 0x04, 0x63, 0x70, 0x75, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x50, 0x55, 0x52, 0x04, 0x63, + 0x70, 0x75, 0x73, 0x12, 0x38, 0x0a, 0x06, 0x63, 0x61, 0x63, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x44, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x52, 0x06, 0x63, 0x61, 0x63, 0x68, 0x65, 0x73, 0x12, 0x48, 0x0a, + 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x50, 0x55, 0x43, 0x61, 0x70, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x92, 0x01, 0x0a, 0x10, 0x4e, 0x6f, 0x64, 0x65, + 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x28, 0x0a, 0x10, + 0x69, 0x73, 0x6f, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x69, 0x64, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0e, 0x69, 0x73, 0x6f, 0x6c, 0x61, 0x74, 0x65, 0x64, + 0x43, 0x70, 0x75, 0x49, 0x64, 0x73, 0x12, 0x29, 0x0a, 0x11, 0x6e, 0x6f, 0x68, 0x7a, 0x5f, 0x66, + 0x75, 0x6c, 0x6c, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0d, 0x52, 0x0e, 0x6e, 0x6f, 0x68, 0x7a, 0x46, 0x75, 0x6c, 0x6c, 0x43, 0x70, 0x75, 0x49, 0x64, + 0x73, 0x12, 0x29, 0x0a, 0x11, 0x72, 0x63, 0x75, 0x5f, 0x6e, 0x6f, 0x63, 0x62, 0x73, 0x5f, 0x63, + 0x70, 0x75, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0e, 0x72, 0x63, + 0x75, 0x4e, 0x6f, 0x63, 0x62, 0x73, 0x43, 0x70, 0x75, 0x49, 0x64, 0x73, 0x22, 0x91, 0x01, 0x0a, + 0x03, 0x54, 0x50, 0x4d, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x22, + 0x0a, 0x0c, 0x6d, 0x61, 0x6e, 0x75, 0x66, 0x61, 0x63, 0x74, 0x75, 0x72, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6d, 0x61, 0x6e, 0x75, 0x66, 0x61, 0x63, 0x74, 0x75, 0x72, + 0x65, 0x72, 0x12, 0x29, 0x0a, 0x10, 0x66, 0x69, 0x72, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x66, 0x69, + 0x72, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, + 0x0c, 0x73, 0x70, 0x65, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x70, 0x65, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x22, 0x90, 0x07, 0x0a, 0x11, 0x48, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x76, + 0x65, 0x6e, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x3f, 0x0a, 0x0b, 0x70, 0x63, 0x69, 0x5f, 0x64, 0x65, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x50, 0x43, 0x49, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x52, 0x0a, 0x70, 0x63, 0x69, + 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x3f, 0x0a, 0x0b, 0x75, 0x73, 0x62, 0x5f, 0x64, + 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x55, 0x53, 0x42, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x52, 0x0a, 0x75, 0x73, + 0x62, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x46, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x69, + 0x61, 0x6c, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, + 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x50, 0x6f, 0x72, + 0x74, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x12, 0x4b, 0x0a, 0x0f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x64, 0x65, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x52, 0x0e, 0x6e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x3f, 0x0a, + 0x0b, 0x63, 0x61, 0x6e, 0x5f, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, + 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x41, 0x4e, 0x44, 0x65, 0x76, 0x69, + 0x63, 0x65, 0x52, 0x0a, 0x63, 0x61, 0x6e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x2d, + 0x0a, 0x04, 0x62, 0x69, 0x6f, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x42, 0x49, 0x4f, 0x53, 0x52, 0x04, 0x62, 0x69, 0x6f, 0x73, 0x12, 0x37, 0x0a, + 0x08, 0x63, 0x70, 0x75, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, + 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x50, 0x55, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x63, + 0x70, 0x75, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x2c, 0x0a, 0x12, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, + 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x10, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x11, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, + 0x79, 0x74, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x77, 0x61, 0x74, 0x63, 0x68, 0x64, 0x6f, 0x67, + 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, + 0x77, 0x61, 0x74, 0x63, 0x68, 0x64, 0x6f, 0x67, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, + 0x2a, 0x0a, 0x03, 0x74, 0x70, 0x6d, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x54, 0x50, 0x4d, 0x52, 0x03, 0x74, 0x70, 0x6d, 0x12, 0x2c, 0x0a, 0x12, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x6c, 0x65, 0x64, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, + 0x74, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4c, + 0x65, 0x64, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x52, 0x0a, 0x11, 0x6e, 0x6f, 0x64, + 0x65, 0x5f, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x10, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x43, + 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x10, 0x6e, 0x6f, 0x64, + 0x65, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x45, 0x0a, + 0x04, 0x6d, 0x69, 0x73, 0x63, 0x18, 0xe8, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x48, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x76, 0x65, 0x6e, + 0x74, 0x6f, 0x72, 0x79, 0x2e, 0x4d, 0x69, 0x73, 0x63, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, + 0x6d, 0x69, 0x73, 0x63, 0x1a, 0x37, 0x0a, 0x09, 0x4d, 0x69, 0x73, 0x63, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, + 0x0c, 0x10, 0x0d, 0x2a, 0xd2, 0x01, 0x0a, 0x11, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x44, + 0x65, 0x76, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x4e, 0x45, 0x54, + 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x4e, 0x45, + 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x45, 0x54, 0x48, 0x45, 0x52, 0x4e, 0x45, 0x54, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, + 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x57, 0x49, 0x46, 0x49, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x4e, 0x45, + 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x57, 0x57, 0x41, 0x4e, 0x10, 0x06, 0x12, 0x1e, 0x0a, 0x1a, 0x4e, 0x45, 0x54, 0x57, + 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x45, 0x54, 0x48, 0x5f, 0x50, 0x46, 0x10, 0x0b, 0x12, 0x1e, 0x0a, 0x1a, 0x4e, 0x45, 0x54, 0x57, + 0x4f, 0x52, 0x4b, 0x5f, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x45, 0x54, 0x48, 0x5f, 0x56, 0x46, 0x10, 0x0c, 0x2a, 0x78, 0x0a, 0x09, 0x43, 0x6f, 0x72, 0x65, + 0x43, 0x6c, 0x61, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x16, 0x43, 0x4f, 0x52, 0x45, 0x5f, 0x43, 0x4c, + 0x41, 0x53, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x43, 0x4f, 0x52, 0x45, 0x5f, 0x43, 0x4c, 0x41, 0x53, 0x53, 0x5f, + 0x50, 0x45, 0x52, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x4e, 0x43, 0x45, 0x10, 0x01, 0x12, 0x19, 0x0a, + 0x15, 0x43, 0x4f, 0x52, 0x45, 0x5f, 0x43, 0x4c, 0x41, 0x53, 0x53, 0x5f, 0x45, 0x46, 0x46, 0x49, + 0x43, 0x49, 0x45, 0x4e, 0x43, 0x59, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x52, 0x45, + 0x5f, 0x43, 0x4c, 0x41, 0x53, 0x53, 0x5f, 0x4c, 0x4f, 0x57, 0x5f, 0x50, 0x4f, 0x57, 0x45, 0x52, + 0x10, 0x03, 0x2a, 0x7b, 0x0a, 0x0a, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4c, 0x65, 0x76, 0x65, 0x6c, + 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x41, 0x43, 0x48, 0x45, 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, + 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x13, 0x0a, + 0x0f, 0x43, 0x41, 0x43, 0x48, 0x45, 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x4c, 0x31, 0x44, + 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x43, 0x41, 0x43, 0x48, 0x45, 0x5f, 0x4c, 0x45, 0x56, 0x45, + 0x4c, 0x5f, 0x4c, 0x31, 0x49, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x41, 0x43, 0x48, 0x45, + 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x4c, 0x32, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x43, + 0x41, 0x43, 0x48, 0x45, 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x4c, 0x33, 0x10, 0x04, 0x42, 0x39, 0x0a, 0x13, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x5a, 0x22, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x66, 0x2d, 0x65, 0x64, 0x67, 0x65, 0x2f, 0x65, 0x76, 0x65, 0x2d, 0x61, @@ -1327,53 +1858,63 @@ func file_info_hardware_proto_rawDescGZIP() []byte { return file_info_hardware_proto_rawDescData } -var file_info_hardware_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_info_hardware_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_info_hardware_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_info_hardware_proto_msgTypes = make([]protoimpl.MessageInfo, 18) var file_info_hardware_proto_goTypes = []interface{}{ (NetworkDeviceType)(0), // 0: org.lfedge.eve.info.NetworkDeviceType - (*USBAddress)(nil), // 1: org.lfedge.eve.info.USBAddress - (*BusParent)(nil), // 2: org.lfedge.eve.info.BusParent - (*USBDevice)(nil), // 3: org.lfedge.eve.info.USBDevice - (*PCIAddress)(nil), // 4: org.lfedge.eve.info.PCIAddress - (*PCIDevice)(nil), // 5: org.lfedge.eve.info.PCIDevice - (*SerialPort)(nil), // 6: org.lfedge.eve.info.SerialPort - (*CANDevice)(nil), // 7: org.lfedge.eve.info.CANDevice - (*NetworkDevice)(nil), // 8: org.lfedge.eve.info.NetworkDevice - (*BIOS)(nil), // 9: org.lfedge.eve.info.BIOS - (*CPU)(nil), // 10: org.lfedge.eve.info.CPU - (*CPUInfo)(nil), // 11: org.lfedge.eve.info.CPUInfo - (*TPM)(nil), // 12: org.lfedge.eve.info.TPM - (*HardwareInventory)(nil), // 13: org.lfedge.eve.info.HardwareInventory - nil, // 14: org.lfedge.eve.info.BIOS.AttributesEntry - nil, // 15: org.lfedge.eve.info.HardwareInventory.MiscEntry + (CoreClass)(0), // 1: org.lfedge.eve.info.CoreClass + (CacheLevel)(0), // 2: org.lfedge.eve.info.CacheLevel + (*USBAddress)(nil), // 3: org.lfedge.eve.info.USBAddress + (*BusParent)(nil), // 4: org.lfedge.eve.info.BusParent + (*USBDevice)(nil), // 5: org.lfedge.eve.info.USBDevice + (*PCIAddress)(nil), // 6: org.lfedge.eve.info.PCIAddress + (*PCIDevice)(nil), // 7: org.lfedge.eve.info.PCIDevice + (*SerialPort)(nil), // 8: org.lfedge.eve.info.SerialPort + (*CANDevice)(nil), // 9: org.lfedge.eve.info.CANDevice + (*NetworkDevice)(nil), // 10: org.lfedge.eve.info.NetworkDevice + (*BIOS)(nil), // 11: org.lfedge.eve.info.BIOS + (*CPU)(nil), // 12: org.lfedge.eve.info.CPU + (*CacheDomain)(nil), // 13: org.lfedge.eve.info.CacheDomain + (*CPUCapabilities)(nil), // 14: org.lfedge.eve.info.CPUCapabilities + (*CPUInfo)(nil), // 15: org.lfedge.eve.info.CPUInfo + (*NodeCapabilities)(nil), // 16: org.lfedge.eve.info.NodeCapabilities + (*TPM)(nil), // 17: org.lfedge.eve.info.TPM + (*HardwareInventory)(nil), // 18: org.lfedge.eve.info.HardwareInventory + nil, // 19: org.lfedge.eve.info.BIOS.AttributesEntry + nil, // 20: org.lfedge.eve.info.HardwareInventory.MiscEntry } var file_info_hardware_proto_depIdxs = []int32{ - 4, // 0: org.lfedge.eve.info.BusParent.pci_parent:type_name -> org.lfedge.eve.info.PCIAddress - 1, // 1: org.lfedge.eve.info.BusParent.usb_parent:type_name -> org.lfedge.eve.info.USBAddress - 2, // 2: org.lfedge.eve.info.USBDevice.parent:type_name -> org.lfedge.eve.info.BusParent - 1, // 3: org.lfedge.eve.info.USBDevice.bus_port:type_name -> org.lfedge.eve.info.USBAddress - 4, // 4: org.lfedge.eve.info.PCIDevice.parent_pci_device_address:type_name -> org.lfedge.eve.info.PCIAddress - 4, // 5: org.lfedge.eve.info.PCIDevice.address:type_name -> org.lfedge.eve.info.PCIAddress - 2, // 6: org.lfedge.eve.info.SerialPort.parent:type_name -> org.lfedge.eve.info.BusParent - 2, // 7: org.lfedge.eve.info.CANDevice.parent:type_name -> org.lfedge.eve.info.BusParent - 2, // 8: org.lfedge.eve.info.NetworkDevice.parent:type_name -> org.lfedge.eve.info.BusParent + 6, // 0: org.lfedge.eve.info.BusParent.pci_parent:type_name -> org.lfedge.eve.info.PCIAddress + 3, // 1: org.lfedge.eve.info.BusParent.usb_parent:type_name -> org.lfedge.eve.info.USBAddress + 4, // 2: org.lfedge.eve.info.USBDevice.parent:type_name -> org.lfedge.eve.info.BusParent + 3, // 3: org.lfedge.eve.info.USBDevice.bus_port:type_name -> org.lfedge.eve.info.USBAddress + 6, // 4: org.lfedge.eve.info.PCIDevice.parent_pci_device_address:type_name -> org.lfedge.eve.info.PCIAddress + 6, // 5: org.lfedge.eve.info.PCIDevice.address:type_name -> org.lfedge.eve.info.PCIAddress + 4, // 6: org.lfedge.eve.info.SerialPort.parent:type_name -> org.lfedge.eve.info.BusParent + 4, // 7: org.lfedge.eve.info.CANDevice.parent:type_name -> org.lfedge.eve.info.BusParent + 4, // 8: org.lfedge.eve.info.NetworkDevice.parent:type_name -> org.lfedge.eve.info.BusParent 0, // 9: org.lfedge.eve.info.NetworkDevice.type:type_name -> org.lfedge.eve.info.NetworkDeviceType - 14, // 10: org.lfedge.eve.info.BIOS.attributes:type_name -> org.lfedge.eve.info.BIOS.AttributesEntry - 10, // 11: org.lfedge.eve.info.CPUInfo.cpus:type_name -> org.lfedge.eve.info.CPU - 5, // 12: org.lfedge.eve.info.HardwareInventory.pci_devices:type_name -> org.lfedge.eve.info.PCIDevice - 3, // 13: org.lfedge.eve.info.HardwareInventory.usb_devices:type_name -> org.lfedge.eve.info.USBDevice - 6, // 14: org.lfedge.eve.info.HardwareInventory.serial_devices:type_name -> org.lfedge.eve.info.SerialPort - 8, // 15: org.lfedge.eve.info.HardwareInventory.network_devices:type_name -> org.lfedge.eve.info.NetworkDevice - 7, // 16: org.lfedge.eve.info.HardwareInventory.can_devices:type_name -> org.lfedge.eve.info.CANDevice - 9, // 17: org.lfedge.eve.info.HardwareInventory.bios:type_name -> org.lfedge.eve.info.BIOS - 11, // 18: org.lfedge.eve.info.HardwareInventory.cpu_info:type_name -> org.lfedge.eve.info.CPUInfo - 12, // 19: org.lfedge.eve.info.HardwareInventory.tpm:type_name -> org.lfedge.eve.info.TPM - 15, // 20: org.lfedge.eve.info.HardwareInventory.misc:type_name -> org.lfedge.eve.info.HardwareInventory.MiscEntry - 21, // [21:21] is the sub-list for method output_type - 21, // [21:21] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name + 19, // 10: org.lfedge.eve.info.BIOS.attributes:type_name -> org.lfedge.eve.info.BIOS.AttributesEntry + 1, // 11: org.lfedge.eve.info.CPU.core_class:type_name -> org.lfedge.eve.info.CoreClass + 2, // 12: org.lfedge.eve.info.CacheDomain.level:type_name -> org.lfedge.eve.info.CacheLevel + 12, // 13: org.lfedge.eve.info.CPUInfo.cpus:type_name -> org.lfedge.eve.info.CPU + 13, // 14: org.lfedge.eve.info.CPUInfo.caches:type_name -> org.lfedge.eve.info.CacheDomain + 14, // 15: org.lfedge.eve.info.CPUInfo.capabilities:type_name -> org.lfedge.eve.info.CPUCapabilities + 7, // 16: org.lfedge.eve.info.HardwareInventory.pci_devices:type_name -> org.lfedge.eve.info.PCIDevice + 5, // 17: org.lfedge.eve.info.HardwareInventory.usb_devices:type_name -> org.lfedge.eve.info.USBDevice + 8, // 18: org.lfedge.eve.info.HardwareInventory.serial_devices:type_name -> org.lfedge.eve.info.SerialPort + 10, // 19: org.lfedge.eve.info.HardwareInventory.network_devices:type_name -> org.lfedge.eve.info.NetworkDevice + 9, // 20: org.lfedge.eve.info.HardwareInventory.can_devices:type_name -> org.lfedge.eve.info.CANDevice + 11, // 21: org.lfedge.eve.info.HardwareInventory.bios:type_name -> org.lfedge.eve.info.BIOS + 15, // 22: org.lfedge.eve.info.HardwareInventory.cpu_info:type_name -> org.lfedge.eve.info.CPUInfo + 17, // 23: org.lfedge.eve.info.HardwareInventory.tpm:type_name -> org.lfedge.eve.info.TPM + 16, // 24: org.lfedge.eve.info.HardwareInventory.node_capabilities:type_name -> org.lfedge.eve.info.NodeCapabilities + 20, // 25: org.lfedge.eve.info.HardwareInventory.misc:type_name -> org.lfedge.eve.info.HardwareInventory.MiscEntry + 26, // [26:26] is the sub-list for method output_type + 26, // [26:26] is the sub-list for method input_type + 26, // [26:26] is the sub-list for extension type_name + 26, // [26:26] is the sub-list for extension extendee + 0, // [0:26] is the sub-list for field type_name } func init() { file_info_hardware_proto_init() } @@ -1503,7 +2044,7 @@ func file_info_hardware_proto_init() { } } file_info_hardware_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CPUInfo); i { + switch v := v.(*CacheDomain); i { case 0: return &v.state case 1: @@ -1515,7 +2056,7 @@ func file_info_hardware_proto_init() { } } file_info_hardware_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TPM); i { + switch v := v.(*CPUCapabilities); i { case 0: return &v.state case 1: @@ -1527,6 +2068,42 @@ func file_info_hardware_proto_init() { } } file_info_hardware_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CPUInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_info_hardware_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeCapabilities); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_info_hardware_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TPM); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_info_hardware_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*HardwareInventory); i { case 0: return &v.state @@ -1550,8 +2127,8 @@ func file_info_hardware_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_info_hardware_proto_rawDesc, - NumEnums: 1, - NumMessages: 15, + NumEnums: 3, + NumMessages: 18, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/info.pb.go b/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/info.pb.go index eef207d82fc..a1993a24d15 100644 --- a/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/info.pb.go +++ b/pkg/pillar/vendor/github.com/lf-edge/eve-api/go/info/info.pb.go @@ -870,6 +870,67 @@ func (StorageTypeInfo) EnumDescriptor() ([]byte, []int) { return file_info_info_proto_rawDescGZIP(), []int{12} } +// CPUPoolKind identifies one partition of the node's logical CPUs. The +// pools are disjoint and together cover the online CPUs. +type CPUPoolKind int32 + +const ( + CPUPoolKind_CPU_POOL_KIND_UNSPECIFIED CPUPoolKind = 0 + // CPUs backing EVE's own housekeeping plus every workload that did not + // ask for dedicated placement. Never empty, so the device stays + // manageable. + CPUPoolKind_CPU_POOL_KIND_HOUSEKEEPING CPUPoolKind = 1 + // CPUs handed out exclusively to workloads with a dedicated CPU policy + // (including SMT siblings parked idle by a one-thread-per-core request). + CPUPoolKind_CPU_POOL_KIND_DEDICATED CPUPoolKind = 2 + // CPUs the running kernel isolates (isolcpus); a subset that dedicated + // workloads requesting the hard isolation tier can be placed into. + CPUPoolKind_CPU_POOL_KIND_ISOLATED CPUPoolKind = 3 +) + +// Enum value maps for CPUPoolKind. +var ( + CPUPoolKind_name = map[int32]string{ + 0: "CPU_POOL_KIND_UNSPECIFIED", + 1: "CPU_POOL_KIND_HOUSEKEEPING", + 2: "CPU_POOL_KIND_DEDICATED", + 3: "CPU_POOL_KIND_ISOLATED", + } + CPUPoolKind_value = map[string]int32{ + "CPU_POOL_KIND_UNSPECIFIED": 0, + "CPU_POOL_KIND_HOUSEKEEPING": 1, + "CPU_POOL_KIND_DEDICATED": 2, + "CPU_POOL_KIND_ISOLATED": 3, + } +) + +func (x CPUPoolKind) Enum() *CPUPoolKind { + p := new(CPUPoolKind) + *p = x + return p +} + +func (x CPUPoolKind) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CPUPoolKind) Descriptor() protoreflect.EnumDescriptor { + return file_info_info_proto_enumTypes[13].Descriptor() +} + +func (CPUPoolKind) Type() protoreflect.EnumType { + return &file_info_info_proto_enumTypes[13] +} + +func (x CPUPoolKind) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CPUPoolKind.Descriptor instead. +func (CPUPoolKind) EnumDescriptor() ([]byte, []int) { + return file_info_info_proto_rawDescGZIP(), []int{13} +} + // Capabilities indicates features in the EdgeDevConfig where there is // no easy way to otherwise determine whether or not they are parsed and // supported by EVE-OS @@ -901,6 +962,7 @@ const ( APICapability_API_CAPABILITY_SMART_REPORT APICapability = 20 // Support for S.M.A.R.T. info on physical storage devices APICapability_API_CAPABILITY_REPORT_TPM_EVENTLOG APICapability = 21 // Support for reporting TPM Event Log "as is" without parsing and selectively reporting events APICapability_API_CAPABILITY_APP_INSTANCE_NET_INTERFACE_CHANGE APICapability = 22 // Support for adding and removing network interfaces without purge, but restart or the edge app + APICapability_API_CAPABILITY_CPU_PLACEMENT_POLICY APICapability = 23 // VmConfig CPU placement policy fields (cpu_policy, full_pcpus_only, threads_per_core, numa_policy, io_placement, isolation_tier, disruption_policy) are parsed and honored ) // Enum value maps for APICapability. @@ -929,6 +991,7 @@ var ( 20: "API_CAPABILITY_SMART_REPORT", 21: "API_CAPABILITY_REPORT_TPM_EVENTLOG", 22: "API_CAPABILITY_APP_INSTANCE_NET_INTERFACE_CHANGE", + 23: "API_CAPABILITY_CPU_PLACEMENT_POLICY", } APICapability_value = map[string]int32{ "API_CAPABILITY_UNSPECIFIED": 0, @@ -954,6 +1017,7 @@ var ( "API_CAPABILITY_SMART_REPORT": 20, "API_CAPABILITY_REPORT_TPM_EVENTLOG": 21, "API_CAPABILITY_APP_INSTANCE_NET_INTERFACE_CHANGE": 22, + "API_CAPABILITY_CPU_PLACEMENT_POLICY": 23, } ) @@ -968,11 +1032,11 @@ func (x APICapability) String() string { } func (APICapability) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[13].Descriptor() + return file_info_info_proto_enumTypes[14].Descriptor() } func (APICapability) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[13] + return &file_info_info_proto_enumTypes[14] } func (x APICapability) Number() protoreflect.EnumNumber { @@ -981,7 +1045,7 @@ func (x APICapability) Number() protoreflect.EnumNumber { // Deprecated: Use APICapability.Descriptor instead. func (APICapability) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{13} + return file_info_info_proto_rawDescGZIP(), []int{14} } // Different reasons for a boot/reboot @@ -1058,11 +1122,11 @@ func (x BootReason) String() string { } func (BootReason) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[14].Descriptor() + return file_info_info_proto_enumTypes[15].Descriptor() } func (BootReason) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[14] + return &file_info_info_proto_enumTypes[15] } func (x BootReason) Number() protoreflect.EnumNumber { @@ -1071,7 +1135,7 @@ func (x BootReason) Number() protoreflect.EnumNumber { // Deprecated: Use BootReason.Descriptor instead. func (BootReason) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{14} + return file_info_info_proto_rawDescGZIP(), []int{15} } // Different reasons why we are in maintenance mode @@ -1120,11 +1184,11 @@ func (x MaintenanceModeReason) String() string { } func (MaintenanceModeReason) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[15].Descriptor() + return file_info_info_proto_enumTypes[16].Descriptor() } func (MaintenanceModeReason) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[15] + return &file_info_info_proto_enumTypes[16] } func (x MaintenanceModeReason) Number() protoreflect.EnumNumber { @@ -1133,7 +1197,7 @@ func (x MaintenanceModeReason) Number() protoreflect.EnumNumber { // Deprecated: Use MaintenanceModeReason.Descriptor instead. func (MaintenanceModeReason) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{15} + return file_info_info_proto_rawDescGZIP(), []int{16} } // Different states of attestation process @@ -1187,11 +1251,11 @@ func (x AttestationState) String() string { } func (AttestationState) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[16].Descriptor() + return file_info_info_proto_enumTypes[17].Descriptor() } func (AttestationState) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[16] + return &file_info_info_proto_enumTypes[17] } func (x AttestationState) Number() protoreflect.EnumNumber { @@ -1200,7 +1264,7 @@ func (x AttestationState) Number() protoreflect.EnumNumber { // Deprecated: Use AttestationState.Descriptor instead. func (AttestationState) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{16} + return file_info_info_proto_rawDescGZIP(), []int{17} } // Different types of app instance metadata @@ -1237,11 +1301,11 @@ func (x AppInstMetaDataType) String() string { } func (AppInstMetaDataType) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[17].Descriptor() + return file_info_info_proto_enumTypes[18].Descriptor() } func (AppInstMetaDataType) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[17] + return &file_info_info_proto_enumTypes[18] } func (x AppInstMetaDataType) Number() protoreflect.EnumNumber { @@ -1250,7 +1314,7 @@ func (x AppInstMetaDataType) Number() protoreflect.EnumNumber { // Deprecated: Use AppInstMetaDataType.Descriptor instead. func (AppInstMetaDataType) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{17} + return file_info_info_proto_rawDescGZIP(), []int{18} } type WirelessType int32 @@ -1286,11 +1350,11 @@ func (x WirelessType) String() string { } func (WirelessType) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[18].Descriptor() + return file_info_info_proto_enumTypes[19].Descriptor() } func (WirelessType) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[18] + return &file_info_info_proto_enumTypes[19] } func (x WirelessType) Number() protoreflect.EnumNumber { @@ -1299,7 +1363,7 @@ func (x WirelessType) Number() protoreflect.EnumNumber { // Deprecated: Use WirelessType.Descriptor instead. func (WirelessType) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{18} + return file_info_info_proto_rawDescGZIP(), []int{19} } type BaseOsStatus int32 @@ -1348,11 +1412,11 @@ func (x BaseOsStatus) String() string { } func (BaseOsStatus) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[19].Descriptor() + return file_info_info_proto_enumTypes[20].Descriptor() } func (BaseOsStatus) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[19] + return &file_info_info_proto_enumTypes[20] } func (x BaseOsStatus) Number() protoreflect.EnumNumber { @@ -1361,7 +1425,7 @@ func (x BaseOsStatus) Number() protoreflect.EnumNumber { // Deprecated: Use BaseOsStatus.Descriptor instead. func (BaseOsStatus) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{19} + return file_info_info_proto_rawDescGZIP(), []int{20} } type BaseOsSubStatus int32 @@ -1412,11 +1476,11 @@ func (x BaseOsSubStatus) String() string { } func (BaseOsSubStatus) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[20].Descriptor() + return file_info_info_proto_enumTypes[21].Descriptor() } func (BaseOsSubStatus) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[20] + return &file_info_info_proto_enumTypes[21] } func (x BaseOsSubStatus) Number() protoreflect.EnumNumber { @@ -1425,7 +1489,7 @@ func (x BaseOsSubStatus) Number() protoreflect.EnumNumber { // Deprecated: Use BaseOsSubStatus.Descriptor instead. func (BaseOsSubStatus) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{20} + return file_info_info_proto_rawDescGZIP(), []int{21} } // Type of the snapshot creation reason @@ -1462,11 +1526,11 @@ func (x SnapshotType) String() string { } func (SnapshotType) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[21].Descriptor() + return file_info_info_proto_enumTypes[22].Descriptor() } func (SnapshotType) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[21] + return &file_info_info_proto_enumTypes[22] } func (x SnapshotType) Number() protoreflect.EnumNumber { @@ -1475,7 +1539,7 @@ func (x SnapshotType) Number() protoreflect.EnumNumber { // Deprecated: Use SnapshotType.Descriptor instead. func (SnapshotType) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{21} + return file_info_info_proto_rawDescGZIP(), []int{22} } type ZInfoClusterNodeStatus int32 @@ -1514,11 +1578,11 @@ func (x ZInfoClusterNodeStatus) String() string { } func (ZInfoClusterNodeStatus) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[22].Descriptor() + return file_info_info_proto_enumTypes[23].Descriptor() } func (ZInfoClusterNodeStatus) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[22] + return &file_info_info_proto_enumTypes[23] } func (x ZInfoClusterNodeStatus) Number() protoreflect.EnumNumber { @@ -1527,7 +1591,7 @@ func (x ZInfoClusterNodeStatus) Number() protoreflect.EnumNumber { // Deprecated: Use ZInfoClusterNodeStatus.Descriptor instead. func (ZInfoClusterNodeStatus) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{22} + return file_info_info_proto_rawDescGZIP(), []int{23} } // ipSec state information @@ -1576,11 +1640,11 @@ func (x ZInfoVpnState) String() string { } func (ZInfoVpnState) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[23].Descriptor() + return file_info_info_proto_enumTypes[24].Descriptor() } func (ZInfoVpnState) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[23] + return &file_info_info_proto_enumTypes[24] } func (x ZInfoVpnState) Number() protoreflect.EnumNumber { @@ -1589,7 +1653,7 @@ func (x ZInfoVpnState) Number() protoreflect.EnumNumber { // Deprecated: Use ZInfoVpnState.Descriptor instead. func (ZInfoVpnState) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{23} + return file_info_info_proto_rawDescGZIP(), []int{24} } type ZNetworkInstanceState int32 @@ -1630,11 +1694,11 @@ func (x ZNetworkInstanceState) String() string { } func (ZNetworkInstanceState) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[24].Descriptor() + return file_info_info_proto_enumTypes[25].Descriptor() } func (ZNetworkInstanceState) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[24] + return &file_info_info_proto_enumTypes[25] } func (x ZNetworkInstanceState) Number() protoreflect.EnumNumber { @@ -1643,7 +1707,7 @@ func (x ZNetworkInstanceState) Number() protoreflect.EnumNumber { // Deprecated: Use ZNetworkInstanceState.Descriptor instead. func (ZNetworkInstanceState) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{24} + return file_info_info_proto_rawDescGZIP(), []int{25} } // LocReliability - reliability of location information. @@ -1686,11 +1750,11 @@ func (x LocReliability) String() string { } func (LocReliability) Descriptor() protoreflect.EnumDescriptor { - return file_info_info_proto_enumTypes[25].Descriptor() + return file_info_info_proto_enumTypes[26].Descriptor() } func (LocReliability) Type() protoreflect.EnumType { - return &file_info_info_proto_enumTypes[25] + return &file_info_info_proto_enumTypes[26] } func (x LocReliability) Number() protoreflect.EnumNumber { @@ -1699,7 +1763,7 @@ func (x LocReliability) Number() protoreflect.EnumNumber { // Deprecated: Use LocReliability.Descriptor instead. func (LocReliability) EnumDescriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{25} + return file_info_info_proto_rawDescGZIP(), []int{26} } type SmartAttr struct { @@ -4101,6 +4165,126 @@ func (x *StorageInfo) GetPoolStatusMsg() string { return "" } +// CPUPoolUtilization reports one CPU pool's extent and how much of it is +// still available. +// +// Both the CPU *sets* and the summary counts are reported: the counts +// answer "how much is left" directly, while the sets let a consumer +// compute availability per request shape by grouping them into physical +// cores via the (socket_id, core_id) coordinates in +// HardwareInventory.CPUInfo. That distinction matters — free threads +// sitting on partially-allocated cores cannot satisfy a request for whole +// physical cores, so a single "free" number would answer the question +// wrongly for one of the two request shapes. +type CPUPoolUtilization struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Kind CPUPoolKind `protobuf:"varint,1,opt,name=kind,proto3,enum=org.lfedge.eve.info.CPUPoolKind" json:"kind,omitempty"` + // Every logical CPU in this pool. + CpuIds []uint32 `protobuf:"varint,2,rep,packed,name=cpu_ids,json=cpuIds,proto3" json:"cpu_ids,omitempty"` + // The subset of cpu_ids not currently allocated to any workload. + FreeCpuIds []uint32 `protobuf:"varint,3,rep,packed,name=free_cpu_ids,json=freeCpuIds,proto3" json:"free_cpu_ids,omitempty"` + // Summary counts, derived from the sets above. + TotalThreads uint32 `protobuf:"varint,4,opt,name=total_threads,json=totalThreads,proto3" json:"total_threads,omitempty"` + AllocatedThreads uint32 `protobuf:"varint,5,opt,name=allocated_threads,json=allocatedThreads,proto3" json:"allocated_threads,omitempty"` + FreeThreads uint32 `protobuf:"varint,6,opt,name=free_threads,json=freeThreads,proto3" json:"free_threads,omitempty"` + // Physical cores all of whose SMT siblings lie in this pool. + TotalCores uint32 `protobuf:"varint,7,opt,name=total_cores,json=totalCores,proto3" json:"total_cores,omitempty"` + // Physical cores all of whose SMT siblings are free — the number that + // actually bounds how many whole-core workloads still fit. + FreeWholeCores uint32 `protobuf:"varint,8,opt,name=free_whole_cores,json=freeWholeCores,proto3" json:"free_whole_cores,omitempty"` +} + +func (x *CPUPoolUtilization) Reset() { + *x = CPUPoolUtilization{} + if protoimpl.UnsafeEnabled { + mi := &file_info_info_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CPUPoolUtilization) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CPUPoolUtilization) ProtoMessage() {} + +func (x *CPUPoolUtilization) ProtoReflect() protoreflect.Message { + mi := &file_info_info_proto_msgTypes[28] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CPUPoolUtilization.ProtoReflect.Descriptor instead. +func (*CPUPoolUtilization) Descriptor() ([]byte, []int) { + return file_info_info_proto_rawDescGZIP(), []int{28} +} + +func (x *CPUPoolUtilization) GetKind() CPUPoolKind { + if x != nil { + return x.Kind + } + return CPUPoolKind_CPU_POOL_KIND_UNSPECIFIED +} + +func (x *CPUPoolUtilization) GetCpuIds() []uint32 { + if x != nil { + return x.CpuIds + } + return nil +} + +func (x *CPUPoolUtilization) GetFreeCpuIds() []uint32 { + if x != nil { + return x.FreeCpuIds + } + return nil +} + +func (x *CPUPoolUtilization) GetTotalThreads() uint32 { + if x != nil { + return x.TotalThreads + } + return 0 +} + +func (x *CPUPoolUtilization) GetAllocatedThreads() uint32 { + if x != nil { + return x.AllocatedThreads + } + return 0 +} + +func (x *CPUPoolUtilization) GetFreeThreads() uint32 { + if x != nil { + return x.FreeThreads + } + return 0 +} + +func (x *CPUPoolUtilization) GetTotalCores() uint32 { + if x != nil { + return x.TotalCores + } + return 0 +} + +func (x *CPUPoolUtilization) GetFreeWholeCores() uint32 { + if x != nil { + return x.FreeWholeCores + } + return 0 +} + // Base device info, as discovered by Xen (or OS on bare metal) type ZInfoDevice struct { state protoimpl.MessageState @@ -4197,12 +4381,18 @@ type ZInfoDevice struct { // Device-reported information about certificates currently enrolled // on the device, typically via network enrollment protocols such as SCEP. EnrolledCerts []*CertInfo `protobuf:"bytes,60,rep,name=enrolled_certs,json=enrolledCerts,proto3" json:"enrolled_certs,omitempty"` + // Current CPU partitioning and utilization, one entry per pool. Dynamic + // state, deliberately reported here (change-driven and periodic) rather + // than on the cached hardware inventory. Lets a controller answer + // "will this workload fit?" before a deploy and explain an + // "cpu.placement.insufficient" failure after one. + CpuPools []*CPUPoolUtilization `protobuf:"bytes,61,rep,name=cpu_pools,json=cpuPools,proto3" json:"cpu_pools,omitempty"` } func (x *ZInfoDevice) Reset() { *x = ZInfoDevice{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[28] + mi := &file_info_info_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4215,7 +4405,7 @@ func (x *ZInfoDevice) String() string { func (*ZInfoDevice) ProtoMessage() {} func (x *ZInfoDevice) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[28] + mi := &file_info_info_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4228,7 +4418,7 @@ func (x *ZInfoDevice) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoDevice.ProtoReflect.Descriptor instead. func (*ZInfoDevice) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{28} + return file_info_info_proto_rawDescGZIP(), []int{29} } func (x *ZInfoDevice) GetMachineArch() string { @@ -4611,6 +4801,13 @@ func (x *ZInfoDevice) GetEnrolledCerts() []*CertInfo { return nil } +func (x *ZInfoDevice) GetCpuPools() []*CPUPoolUtilization { + if x != nil { + return x.CpuPools + } + return nil +} + // OptionalCapabilities indicates any additional capabilities device wants // to publish to controller. For example Kubevirt hypervisor is not supported by // all eve flavors. @@ -4624,12 +4821,20 @@ type OptionalCapabilities struct { HwInventorySupport bool `protobuf:"varint,2,opt,name=hw_inventory_support,json=hwInventorySupport,proto3" json:"hw_inventory_support,omitempty"` // Device supports etcd snapshots (e.g. eve-k flavor) EtcdSnapshot bool `protobuf:"varint,3,opt,name=etcd_snapshot,json=etcdSnapshot,proto3" json:"etcd_snapshot,omitempty"` + // EVE is able to derive a CPU-isolation kernel command line + // (isolcpus/nohz_full/rcu_nocbs) from its own CPU placement plan and apply + // it itself, which takes effect on the next boot (reboot-gated). False means + // EVE will only schedule into a statically provisioned isolated pool, if the + // running kernel reports one (see info HardwareInventory.node_capabilities). + // This is an EVE-software ability, not a hardware or kernel fact, which is + // why it lives here and not in the hardware inventory. + ManagedCpuIsolation bool `protobuf:"varint,4,opt,name=managed_cpu_isolation,json=managedCpuIsolation,proto3" json:"managed_cpu_isolation,omitempty"` } func (x *OptionalCapabilities) Reset() { *x = OptionalCapabilities{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[29] + mi := &file_info_info_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4642,7 +4847,7 @@ func (x *OptionalCapabilities) String() string { func (*OptionalCapabilities) ProtoMessage() {} func (x *OptionalCapabilities) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[29] + mi := &file_info_info_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4655,7 +4860,7 @@ func (x *OptionalCapabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use OptionalCapabilities.ProtoReflect.Descriptor instead. func (*OptionalCapabilities) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{29} + return file_info_info_proto_rawDescGZIP(), []int{30} } func (x *OptionalCapabilities) GetHvTypeKubevirt() bool { @@ -4679,6 +4884,13 @@ func (x *OptionalCapabilities) GetEtcdSnapshot() bool { return false } +func (x *OptionalCapabilities) GetManagedCpuIsolation() bool { + if x != nil { + return x.ManagedCpuIsolation + } + return false +} + // Information about attestation process type AttestationInfo struct { state protoimpl.MessageState @@ -4692,7 +4904,7 @@ type AttestationInfo struct { func (x *AttestationInfo) Reset() { *x = AttestationInfo{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[30] + mi := &file_info_info_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4705,7 +4917,7 @@ func (x *AttestationInfo) String() string { func (*AttestationInfo) ProtoMessage() {} func (x *AttestationInfo) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[30] + mi := &file_info_info_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4718,7 +4930,7 @@ func (x *AttestationInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use AttestationInfo.ProtoReflect.Descriptor instead. func (*AttestationInfo) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{30} + return file_info_info_proto_rawDescGZIP(), []int{31} } func (x *AttestationInfo) GetState() AttestationState { @@ -4748,7 +4960,7 @@ type SystemAdapterInfo struct { func (x *SystemAdapterInfo) Reset() { *x = SystemAdapterInfo{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[31] + mi := &file_info_info_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4761,7 +4973,7 @@ func (x *SystemAdapterInfo) String() string { func (*SystemAdapterInfo) ProtoMessage() {} func (x *SystemAdapterInfo) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[31] + mi := &file_info_info_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4774,7 +4986,7 @@ func (x *SystemAdapterInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemAdapterInfo.ProtoReflect.Descriptor instead. func (*SystemAdapterInfo) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{31} + return file_info_info_proto_rawDescGZIP(), []int{32} } func (x *SystemAdapterInfo) GetCurrentIndex() uint32 { @@ -4808,7 +5020,7 @@ type DevicePortStatus struct { func (x *DevicePortStatus) Reset() { *x = DevicePortStatus{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[32] + mi := &file_info_info_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4821,7 +5033,7 @@ func (x *DevicePortStatus) String() string { func (*DevicePortStatus) ProtoMessage() {} func (x *DevicePortStatus) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[32] + mi := &file_info_info_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4834,7 +5046,7 @@ func (x *DevicePortStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use DevicePortStatus.ProtoReflect.Descriptor instead. func (*DevicePortStatus) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{32} + return file_info_info_proto_rawDescGZIP(), []int{33} } func (x *DevicePortStatus) GetVersion() uint32 { @@ -4934,7 +5146,7 @@ type DevicePort struct { func (x *DevicePort) Reset() { *x = DevicePort{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[33] + mi := &file_info_info_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4947,7 +5159,7 @@ func (x *DevicePort) String() string { func (*DevicePort) ProtoMessage() {} func (x *DevicePort) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[33] + mi := &file_info_info_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4960,7 +5172,7 @@ func (x *DevicePort) ProtoReflect() protoreflect.Message { // Deprecated: Use DevicePort.ProtoReflect.Descriptor instead. func (*DevicePort) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{33} + return file_info_info_proto_rawDescGZIP(), []int{34} } func (x *DevicePort) GetIfname() string { @@ -5182,7 +5394,7 @@ type ProxyStatus struct { func (x *ProxyStatus) Reset() { *x = ProxyStatus{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[34] + mi := &file_info_info_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5195,7 +5407,7 @@ func (x *ProxyStatus) String() string { func (*ProxyStatus) ProtoMessage() {} func (x *ProxyStatus) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[34] + mi := &file_info_info_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5208,7 +5420,7 @@ func (x *ProxyStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProxyStatus.ProtoReflect.Descriptor instead. func (*ProxyStatus) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{34} + return file_info_info_proto_rawDescGZIP(), []int{35} } func (x *ProxyStatus) GetProxies() []*ProxyEntry { @@ -5266,7 +5478,7 @@ type ProxyEntry struct { func (x *ProxyEntry) Reset() { *x = ProxyEntry{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[35] + mi := &file_info_info_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5279,7 +5491,7 @@ func (x *ProxyEntry) String() string { func (*ProxyEntry) ProtoMessage() {} func (x *ProxyEntry) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[35] + mi := &file_info_info_proto_msgTypes[36] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5292,7 +5504,7 @@ func (x *ProxyEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use ProxyEntry.ProtoReflect.Descriptor instead. func (*ProxyEntry) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{35} + return file_info_info_proto_rawDescGZIP(), []int{36} } func (x *ProxyEntry) GetType() uint32 { @@ -5328,7 +5540,7 @@ type WirelessStatus struct { func (x *WirelessStatus) Reset() { *x = WirelessStatus{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[36] + mi := &file_info_info_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5341,7 +5553,7 @@ func (x *WirelessStatus) String() string { func (*WirelessStatus) ProtoMessage() {} func (x *WirelessStatus) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[36] + mi := &file_info_info_proto_msgTypes[37] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5354,7 +5566,7 @@ func (x *WirelessStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use WirelessStatus.ProtoReflect.Descriptor instead. func (*WirelessStatus) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{36} + return file_info_info_proto_rawDescGZIP(), []int{37} } func (x *WirelessStatus) GetType() WirelessType { @@ -5407,7 +5619,7 @@ type ZCellularStatus struct { func (x *ZCellularStatus) Reset() { *x = ZCellularStatus{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[37] + mi := &file_info_info_proto_msgTypes[38] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5420,7 +5632,7 @@ func (x *ZCellularStatus) String() string { func (*ZCellularStatus) ProtoMessage() {} func (x *ZCellularStatus) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[37] + mi := &file_info_info_proto_msgTypes[38] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5433,7 +5645,7 @@ func (x *ZCellularStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ZCellularStatus.ProtoReflect.Descriptor instead. func (*ZCellularStatus) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{37} + return file_info_info_proto_rawDescGZIP(), []int{38} } func (x *ZCellularStatus) GetCellularModule() string { @@ -5526,7 +5738,7 @@ type ZInfoDevSW struct { func (x *ZInfoDevSW) Reset() { *x = ZInfoDevSW{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[38] + mi := &file_info_info_proto_msgTypes[39] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5539,7 +5751,7 @@ func (x *ZInfoDevSW) String() string { func (*ZInfoDevSW) ProtoMessage() {} func (x *ZInfoDevSW) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[38] + mi := &file_info_info_proto_msgTypes[39] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5552,7 +5764,7 @@ func (x *ZInfoDevSW) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoDevSW.ProtoReflect.Descriptor instead. func (*ZInfoDevSW) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{38} + return file_info_info_proto_rawDescGZIP(), []int{39} } func (x *ZInfoDevSW) GetActivated() bool { @@ -5677,7 +5889,7 @@ type ZInfoStorage struct { func (x *ZInfoStorage) Reset() { *x = ZInfoStorage{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[39] + mi := &file_info_info_proto_msgTypes[40] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5690,7 +5902,7 @@ func (x *ZInfoStorage) String() string { func (*ZInfoStorage) ProtoMessage() {} func (x *ZInfoStorage) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[39] + mi := &file_info_info_proto_msgTypes[40] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5703,7 +5915,7 @@ func (x *ZInfoStorage) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoStorage.ProtoReflect.Descriptor instead. func (*ZInfoStorage) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{39} + return file_info_info_proto_rawDescGZIP(), []int{40} } func (x *ZInfoStorage) GetDevice() string { @@ -5793,7 +6005,7 @@ type ZInfoSnapshot struct { func (x *ZInfoSnapshot) Reset() { *x = ZInfoSnapshot{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[40] + mi := &file_info_info_proto_msgTypes[41] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5806,7 +6018,7 @@ func (x *ZInfoSnapshot) String() string { func (*ZInfoSnapshot) ProtoMessage() {} func (x *ZInfoSnapshot) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[40] + mi := &file_info_info_proto_msgTypes[41] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5819,7 +6031,7 @@ func (x *ZInfoSnapshot) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoSnapshot.ProtoReflect.Descriptor instead. func (*ZInfoSnapshot) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{40} + return file_info_info_proto_rawDescGZIP(), []int{41} } func (x *ZInfoSnapshot) GetId() string { @@ -5875,7 +6087,7 @@ type ZInfoClusterNode struct { func (x *ZInfoClusterNode) Reset() { *x = ZInfoClusterNode{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[41] + mi := &file_info_info_proto_msgTypes[42] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5888,7 +6100,7 @@ func (x *ZInfoClusterNode) String() string { func (*ZInfoClusterNode) ProtoMessage() {} func (x *ZInfoClusterNode) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[41] + mi := &file_info_info_proto_msgTypes[42] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5901,7 +6113,7 @@ func (x *ZInfoClusterNode) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoClusterNode.ProtoReflect.Descriptor instead. func (*ZInfoClusterNode) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{41} + return file_info_info_proto_rawDescGZIP(), []int{42} } func (x *ZInfoClusterNode) GetNodeStatus() ZInfoClusterNodeStatus { @@ -5937,7 +6149,7 @@ type ZInfoApp struct { func (x *ZInfoApp) Reset() { *x = ZInfoApp{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[42] + mi := &file_info_info_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5950,7 +6162,7 @@ func (x *ZInfoApp) String() string { func (*ZInfoApp) ProtoMessage() {} func (x *ZInfoApp) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[42] + mi := &file_info_info_proto_msgTypes[43] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5963,7 +6175,7 @@ func (x *ZInfoApp) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoApp.ProtoReflect.Descriptor instead. func (*ZInfoApp) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{42} + return file_info_info_proto_rawDescGZIP(), []int{43} } func (x *ZInfoApp) GetAppID() string { @@ -6071,7 +6283,7 @@ type ZInfoVpnLinkInfo struct { func (x *ZInfoVpnLinkInfo) Reset() { *x = ZInfoVpnLinkInfo{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[43] + mi := &file_info_info_proto_msgTypes[44] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6084,7 +6296,7 @@ func (x *ZInfoVpnLinkInfo) String() string { func (*ZInfoVpnLinkInfo) ProtoMessage() {} func (x *ZInfoVpnLinkInfo) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[43] + mi := &file_info_info_proto_msgTypes[44] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6097,7 +6309,7 @@ func (x *ZInfoVpnLinkInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoVpnLinkInfo.ProtoReflect.Descriptor instead. func (*ZInfoVpnLinkInfo) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{43} + return file_info_info_proto_rawDescGZIP(), []int{44} } func (x *ZInfoVpnLinkInfo) GetSpiId() string { @@ -6140,7 +6352,7 @@ type ZInfoVpnLink struct { func (x *ZInfoVpnLink) Reset() { *x = ZInfoVpnLink{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[44] + mi := &file_info_info_proto_msgTypes[45] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6153,7 +6365,7 @@ func (x *ZInfoVpnLink) String() string { func (*ZInfoVpnLink) ProtoMessage() {} func (x *ZInfoVpnLink) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[44] + mi := &file_info_info_proto_msgTypes[45] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6166,7 +6378,7 @@ func (x *ZInfoVpnLink) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoVpnLink.ProtoReflect.Descriptor instead. func (*ZInfoVpnLink) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{44} + return file_info_info_proto_rawDescGZIP(), []int{45} } func (x *ZInfoVpnLink) GetId() string { @@ -6239,7 +6451,7 @@ type ZInfoVpnEndPoint struct { func (x *ZInfoVpnEndPoint) Reset() { *x = ZInfoVpnEndPoint{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[45] + mi := &file_info_info_proto_msgTypes[46] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6252,7 +6464,7 @@ func (x *ZInfoVpnEndPoint) String() string { func (*ZInfoVpnEndPoint) ProtoMessage() {} func (x *ZInfoVpnEndPoint) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[45] + mi := &file_info_info_proto_msgTypes[46] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6265,7 +6477,7 @@ func (x *ZInfoVpnEndPoint) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoVpnEndPoint.ProtoReflect.Descriptor instead. func (*ZInfoVpnEndPoint) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{45} + return file_info_info_proto_rawDescGZIP(), []int{46} } func (x *ZInfoVpnEndPoint) GetId() string { @@ -6309,7 +6521,7 @@ type ZInfoVpnConn struct { func (x *ZInfoVpnConn) Reset() { *x = ZInfoVpnConn{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[46] + mi := &file_info_info_proto_msgTypes[47] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6322,7 +6534,7 @@ func (x *ZInfoVpnConn) String() string { func (*ZInfoVpnConn) ProtoMessage() {} func (x *ZInfoVpnConn) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[46] + mi := &file_info_info_proto_msgTypes[47] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6335,7 +6547,7 @@ func (x *ZInfoVpnConn) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoVpnConn.ProtoReflect.Descriptor instead. func (*ZInfoVpnConn) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{46} + return file_info_info_proto_rawDescGZIP(), []int{47} } func (x *ZInfoVpnConn) GetId() string { @@ -6416,7 +6628,7 @@ type ZInfoVpn struct { func (x *ZInfoVpn) Reset() { *x = ZInfoVpn{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[47] + mi := &file_info_info_proto_msgTypes[48] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6429,7 +6641,7 @@ func (x *ZInfoVpn) String() string { func (*ZInfoVpn) ProtoMessage() {} func (x *ZInfoVpn) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[47] + mi := &file_info_info_proto_msgTypes[48] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6442,7 +6654,7 @@ func (x *ZInfoVpn) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoVpn.ProtoReflect.Descriptor instead. func (*ZInfoVpn) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{47} + return file_info_info_proto_rawDescGZIP(), []int{48} } func (x *ZInfoVpn) GetUpTime() uint64 { @@ -6543,7 +6755,7 @@ type ZInfoNetworkInstance struct { func (x *ZInfoNetworkInstance) Reset() { *x = ZInfoNetworkInstance{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[48] + mi := &file_info_info_proto_msgTypes[49] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6556,7 +6768,7 @@ func (x *ZInfoNetworkInstance) String() string { func (*ZInfoNetworkInstance) ProtoMessage() {} func (x *ZInfoNetworkInstance) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[48] + mi := &file_info_info_proto_msgTypes[49] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6569,7 +6781,7 @@ func (x *ZInfoNetworkInstance) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoNetworkInstance.ProtoReflect.Descriptor instead. func (*ZInfoNetworkInstance) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{48} + return file_info_info_proto_rawDescGZIP(), []int{49} } func (x *ZInfoNetworkInstance) GetNetworkID() string { @@ -6776,7 +6988,7 @@ type IPRoute struct { func (x *IPRoute) Reset() { *x = IPRoute{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[49] + mi := &file_info_info_proto_msgTypes[50] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6789,7 +7001,7 @@ func (x *IPRoute) String() string { func (*IPRoute) ProtoMessage() {} func (x *IPRoute) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[49] + mi := &file_info_info_proto_msgTypes[50] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6802,7 +7014,7 @@ func (x *IPRoute) ProtoReflect() protoreflect.Message { // Deprecated: Use IPRoute.ProtoReflect.Descriptor instead. func (*IPRoute) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{49} + return file_info_info_proto_rawDescGZIP(), []int{50} } func (x *IPRoute) GetDestinationNetwork() string { @@ -6846,7 +7058,7 @@ type UsageInfo struct { func (x *UsageInfo) Reset() { *x = UsageInfo{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[50] + mi := &file_info_info_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6859,7 +7071,7 @@ func (x *UsageInfo) String() string { func (*UsageInfo) ProtoMessage() {} func (x *UsageInfo) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[50] + mi := &file_info_info_proto_msgTypes[51] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6872,7 +7084,7 @@ func (x *UsageInfo) ProtoReflect() protoreflect.Message { // Deprecated: Use UsageInfo.ProtoReflect.Descriptor instead. func (*UsageInfo) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{50} + return file_info_info_proto_rawDescGZIP(), []int{51} } func (x *UsageInfo) GetCreateTime() *timestamppb.Timestamp { @@ -6908,7 +7120,7 @@ type VolumeResources struct { func (x *VolumeResources) Reset() { *x = VolumeResources{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[51] + mi := &file_info_info_proto_msgTypes[52] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6921,7 +7133,7 @@ func (x *VolumeResources) String() string { func (*VolumeResources) ProtoMessage() {} func (x *VolumeResources) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[51] + mi := &file_info_info_proto_msgTypes[52] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6934,7 +7146,7 @@ func (x *VolumeResources) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeResources.ProtoReflect.Descriptor instead. func (*VolumeResources) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{51} + return file_info_info_proto_rawDescGZIP(), []int{52} } func (x *VolumeResources) GetMaxSizeBytes() uint64 { @@ -6979,7 +7191,7 @@ type ZInfoVolume struct { func (x *ZInfoVolume) Reset() { *x = ZInfoVolume{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[52] + mi := &file_info_info_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6992,7 +7204,7 @@ func (x *ZInfoVolume) String() string { func (*ZInfoVolume) ProtoMessage() {} func (x *ZInfoVolume) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[52] + mi := &file_info_info_proto_msgTypes[53] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7005,7 +7217,7 @@ func (x *ZInfoVolume) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoVolume.ProtoReflect.Descriptor instead. func (*ZInfoVolume) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{52} + return file_info_info_proto_rawDescGZIP(), []int{53} } func (x *ZInfoVolume) GetUuid() string { @@ -7075,7 +7287,7 @@ type ContentResources struct { func (x *ContentResources) Reset() { *x = ContentResources{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[53] + mi := &file_info_info_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7088,7 +7300,7 @@ func (x *ContentResources) String() string { func (*ContentResources) ProtoMessage() {} func (x *ContentResources) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[53] + mi := &file_info_info_proto_msgTypes[54] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7101,7 +7313,7 @@ func (x *ContentResources) ProtoReflect() protoreflect.Message { // Deprecated: Use ContentResources.ProtoReflect.Descriptor instead. func (*ContentResources) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{53} + return file_info_info_proto_rawDescGZIP(), []int{54} } func (x *ContentResources) GetCurSizeBytes() uint64 { @@ -7144,7 +7356,7 @@ type ZInfoContentTree struct { func (x *ZInfoContentTree) Reset() { *x = ZInfoContentTree{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[54] + mi := &file_info_info_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7157,7 +7369,7 @@ func (x *ZInfoContentTree) String() string { func (*ZInfoContentTree) ProtoMessage() {} func (x *ZInfoContentTree) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[54] + mi := &file_info_info_proto_msgTypes[55] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7170,7 +7382,7 @@ func (x *ZInfoContentTree) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoContentTree.ProtoReflect.Descriptor instead. func (*ZInfoContentTree) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{54} + return file_info_info_proto_rawDescGZIP(), []int{55} } func (x *ZInfoContentTree) GetUuid() string { @@ -7268,7 +7480,7 @@ type ZInfoBlob struct { func (x *ZInfoBlob) Reset() { *x = ZInfoBlob{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[55] + mi := &file_info_info_proto_msgTypes[56] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7281,7 +7493,7 @@ func (x *ZInfoBlob) String() string { func (*ZInfoBlob) ProtoMessage() {} func (x *ZInfoBlob) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[55] + mi := &file_info_info_proto_msgTypes[56] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7294,7 +7506,7 @@ func (x *ZInfoBlob) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoBlob.ProtoReflect.Descriptor instead. func (*ZInfoBlob) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{55} + return file_info_info_proto_rawDescGZIP(), []int{56} } func (x *ZInfoBlob) GetSha256() string { @@ -7351,7 +7563,7 @@ type ZInfoBlobList struct { func (x *ZInfoBlobList) Reset() { *x = ZInfoBlobList{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[56] + mi := &file_info_info_proto_msgTypes[57] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7364,7 +7576,7 @@ func (x *ZInfoBlobList) String() string { func (*ZInfoBlobList) ProtoMessage() {} func (x *ZInfoBlobList) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[56] + mi := &file_info_info_proto_msgTypes[57] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7377,7 +7589,7 @@ func (x *ZInfoBlobList) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoBlobList.ProtoReflect.Descriptor instead. func (*ZInfoBlobList) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{56} + return file_info_info_proto_rawDescGZIP(), []int{57} } func (x *ZInfoBlobList) GetBlob() []*ZInfoBlob { @@ -7424,7 +7636,7 @@ type ZInfoMsg struct { func (x *ZInfoMsg) Reset() { *x = ZInfoMsg{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[57] + mi := &file_info_info_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7437,7 +7649,7 @@ func (x *ZInfoMsg) String() string { func (*ZInfoMsg) ProtoMessage() {} func (x *ZInfoMsg) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[57] + mi := &file_info_info_proto_msgTypes[58] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7450,7 +7662,7 @@ func (x *ZInfoMsg) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoMsg.ProtoReflect.Descriptor instead. func (*ZInfoMsg) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{57} + return file_info_info_proto_rawDescGZIP(), []int{58} } func (x *ZInfoMsg) GetZtype() ZInfoTypes { @@ -7699,7 +7911,7 @@ type Capabilities struct { func (x *Capabilities) Reset() { *x = Capabilities{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[58] + mi := &file_info_info_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7712,7 +7924,7 @@ func (x *Capabilities) String() string { func (*Capabilities) ProtoMessage() {} func (x *Capabilities) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[58] + mi := &file_info_info_proto_msgTypes[59] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7725,7 +7937,7 @@ func (x *Capabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use Capabilities.ProtoReflect.Descriptor instead. func (*Capabilities) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{58} + return file_info_info_proto_rawDescGZIP(), []int{59} } func (x *Capabilities) GetHWAssistedVirtualization() bool { @@ -7758,7 +7970,7 @@ type ZInfoAppInstMetaData struct { func (x *ZInfoAppInstMetaData) Reset() { *x = ZInfoAppInstMetaData{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[59] + mi := &file_info_info_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7771,7 +7983,7 @@ func (x *ZInfoAppInstMetaData) String() string { func (*ZInfoAppInstMetaData) ProtoMessage() {} func (x *ZInfoAppInstMetaData) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[59] + mi := &file_info_info_proto_msgTypes[60] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7784,7 +7996,7 @@ func (x *ZInfoAppInstMetaData) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoAppInstMetaData.ProtoReflect.Descriptor instead. func (*ZInfoAppInstMetaData) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{59} + return file_info_info_proto_rawDescGZIP(), []int{60} } func (x *ZInfoAppInstMetaData) GetUuid() string { @@ -7829,7 +8041,7 @@ type ZInfoEdgeview struct { func (x *ZInfoEdgeview) Reset() { *x = ZInfoEdgeview{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[60] + mi := &file_info_info_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7842,7 +8054,7 @@ func (x *ZInfoEdgeview) String() string { func (*ZInfoEdgeview) ProtoMessage() {} func (x *ZInfoEdgeview) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[60] + mi := &file_info_info_proto_msgTypes[61] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7855,7 +8067,7 @@ func (x *ZInfoEdgeview) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoEdgeview.ProtoReflect.Descriptor instead. func (*ZInfoEdgeview) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{60} + return file_info_info_proto_rawDescGZIP(), []int{61} } func (x *ZInfoEdgeview) GetExpireTime() *timestamppb.Timestamp { @@ -7939,7 +8151,7 @@ type ZInfoLocation struct { func (x *ZInfoLocation) Reset() { *x = ZInfoLocation{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[61] + mi := &file_info_info_proto_msgTypes[62] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -7952,7 +8164,7 @@ func (x *ZInfoLocation) String() string { func (*ZInfoLocation) ProtoMessage() {} func (x *ZInfoLocation) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[61] + mi := &file_info_info_proto_msgTypes[62] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -7965,7 +8177,7 @@ func (x *ZInfoLocation) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoLocation.ProtoReflect.Descriptor instead. func (*ZInfoLocation) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{61} + return file_info_info_proto_rawDescGZIP(), []int{62} } func (x *ZInfoLocation) GetLatitude() float64 { @@ -8055,7 +8267,7 @@ type ZInfoKubeClusterUpdateStatus struct { func (x *ZInfoKubeClusterUpdateStatus) Reset() { *x = ZInfoKubeClusterUpdateStatus{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[62] + mi := &file_info_info_proto_msgTypes[63] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8068,7 +8280,7 @@ func (x *ZInfoKubeClusterUpdateStatus) String() string { func (*ZInfoKubeClusterUpdateStatus) ProtoMessage() {} func (x *ZInfoKubeClusterUpdateStatus) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[62] + mi := &file_info_info_proto_msgTypes[63] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8081,7 +8293,7 @@ func (x *ZInfoKubeClusterUpdateStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoKubeClusterUpdateStatus.ProtoReflect.Descriptor instead. func (*ZInfoKubeClusterUpdateStatus) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{62} + return file_info_info_proto_rawDescGZIP(), []int{63} } func (x *ZInfoKubeClusterUpdateStatus) GetCurrentNode() string { @@ -8143,7 +8355,7 @@ type ZInfoKubeCluster struct { func (x *ZInfoKubeCluster) Reset() { *x = ZInfoKubeCluster{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[63] + mi := &file_info_info_proto_msgTypes[64] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8156,7 +8368,7 @@ func (x *ZInfoKubeCluster) String() string { func (*ZInfoKubeCluster) ProtoMessage() {} func (x *ZInfoKubeCluster) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[63] + mi := &file_info_info_proto_msgTypes[64] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8169,7 +8381,7 @@ func (x *ZInfoKubeCluster) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoKubeCluster.ProtoReflect.Descriptor instead. func (*ZInfoKubeCluster) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{63} + return file_info_info_proto_rawDescGZIP(), []int{64} } func (x *ZInfoKubeCluster) GetNodes() []*KubeNodeInfo { @@ -8238,7 +8450,7 @@ type ZInfoHardware struct { func (x *ZInfoHardware) Reset() { *x = ZInfoHardware{} if protoimpl.UnsafeEnabled { - mi := &file_info_info_proto_msgTypes[64] + mi := &file_info_info_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -8251,7 +8463,7 @@ func (x *ZInfoHardware) String() string { func (*ZInfoHardware) ProtoMessage() {} func (x *ZInfoHardware) ProtoReflect() protoreflect.Message { - mi := &file_info_info_proto_msgTypes[64] + mi := &file_info_info_proto_msgTypes[65] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -8264,7 +8476,7 @@ func (x *ZInfoHardware) ProtoReflect() protoreflect.Message { // Deprecated: Use ZInfoHardware.ProtoReflect.Descriptor instead. func (*ZInfoHardware) Descriptor() ([]byte, []int) { - return file_info_info_proto_rawDescGZIP(), []int{64} + return file_info_info_proto_rawDescGZIP(), []int{65} } // Deprecated: Marked as deprecated in info/info.proto. @@ -8751,1298 +8963,1337 @@ var file_info_info_proto_rawDesc = []byte{ 0x6e, 0x52, 0x08, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x12, 0x26, 0x0a, 0x0f, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x6f, 0x6f, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x4d, 0x73, 0x67, 0x22, 0xe0, 0x17, 0x0a, 0x0b, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x41, 0x72, - 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, - 0x65, 0x41, 0x72, 0x63, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x70, 0x75, 0x41, 0x72, 0x63, 0x68, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x70, 0x75, 0x41, 0x72, 0x63, 0x68, 0x12, - 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x63, 0x70, 0x75, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6e, 0x63, 0x70, 0x75, 0x12, - 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x74, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, - 0x65, 0x12, 0x2c, 0x0a, 0x11, 0x70, 0x6f, 0x77, 0x65, 0x72, 0x43, 0x79, 0x63, 0x6c, 0x65, 0x43, - 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x70, 0x6f, - 0x77, 0x65, 0x72, 0x43, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, - 0x3c, 0x0a, 0x05, 0x6d, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, - 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4d, 0x61, 0x6e, 0x75, 0x66, 0x61, - 0x63, 0x74, 0x75, 0x72, 0x65, 0x72, 0x52, 0x05, 0x6d, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, - 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, + 0x4d, 0x73, 0x67, 0x22, 0xc5, 0x02, 0x0a, 0x12, 0x43, 0x50, 0x55, 0x50, 0x6f, 0x6f, 0x6c, 0x55, + 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x34, 0x0a, 0x04, 0x6b, 0x69, + 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, + 0x50, 0x55, 0x50, 0x6f, 0x6f, 0x6c, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, + 0x12, 0x17, 0x0a, 0x07, 0x63, 0x70, 0x75, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0d, 0x52, 0x06, 0x63, 0x70, 0x75, 0x49, 0x64, 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x66, 0x72, 0x65, + 0x65, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0d, 0x52, + 0x0a, 0x66, 0x72, 0x65, 0x65, 0x43, 0x70, 0x75, 0x49, 0x64, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x74, + 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, + 0x12, 0x2b, 0x0a, 0x11, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x68, + 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x61, 0x6c, 0x6c, + 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x21, 0x0a, + 0x0c, 0x66, 0x72, 0x65, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x66, 0x72, 0x65, 0x65, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, + 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x72, 0x65, + 0x73, 0x12, 0x28, 0x0a, 0x10, 0x66, 0x72, 0x65, 0x65, 0x5f, 0x77, 0x68, 0x6f, 0x6c, 0x65, 0x5f, + 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x66, 0x72, 0x65, + 0x65, 0x57, 0x68, 0x6f, 0x6c, 0x65, 0x43, 0x6f, 0x72, 0x65, 0x73, 0x22, 0xa6, 0x18, 0x0a, 0x0b, + 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x6d, + 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x41, 0x72, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x41, 0x72, 0x63, 0x68, 0x12, 0x18, 0x0a, + 0x07, 0x63, 0x70, 0x75, 0x41, 0x72, 0x63, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x63, 0x70, 0x75, 0x41, 0x72, 0x63, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, + 0x6f, 0x72, 0x6d, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, + 0x6f, 0x72, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x63, 0x70, 0x75, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x04, 0x6e, 0x63, 0x70, 0x75, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, + 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x07, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x2c, 0x0a, 0x11, 0x70, 0x6f, 0x77, + 0x65, 0x72, 0x43, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x11, 0x70, 0x6f, 0x77, 0x65, 0x72, 0x43, 0x79, 0x63, 0x6c, 0x65, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x3c, 0x0a, 0x05, 0x6d, 0x69, 0x6e, 0x66, 0x6f, + 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, + 0x66, 0x6f, 0x4d, 0x61, 0x6e, 0x75, 0x66, 0x61, 0x63, 0x74, 0x75, 0x72, 0x65, 0x72, 0x52, 0x05, + 0x6d, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, + 0x66, 0x6f, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x12, 0x4e, 0x0a, 0x12, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x73, 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, - 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x4e, 0x0a, 0x12, 0x61, 0x73, - 0x73, 0x69, 0x67, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x73, - 0x18, 0x0f, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x69, 0x6f, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x12, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x03, 0x64, 0x6e, - 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, - 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, - 0x6e, 0x66, 0x6f, 0x44, 0x4e, 0x53, 0x52, 0x03, 0x64, 0x6e, 0x73, 0x12, 0x43, 0x0a, 0x0b, 0x73, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x69, 0x73, 0x74, - 0x12, 0x36, 0x0a, 0x08, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x12, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, - 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x37, 0x0a, 0x06, 0x73, 0x77, 0x4c, 0x69, - 0x73, 0x74, 0x18, 0x13, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x69, 0x6f, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x12, + 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, + 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, + 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x4e, 0x53, 0x52, 0x03, + 0x64, 0x6e, 0x73, 0x12, 0x43, 0x0a, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x69, + 0x73, 0x74, 0x18, 0x11, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, - 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, 0x53, 0x57, 0x52, 0x06, 0x73, 0x77, 0x4c, 0x69, 0x73, - 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x48, 0x6f, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x14, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x48, 0x6f, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x4b, 0x0a, - 0x0b, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, - 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, - 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x0b, 0x6d, - 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x12, 0x2a, 0x0a, 0x10, 0x6c, 0x61, - 0x73, 0x74, 0x52, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x16, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x62, 0x6f, 0x6f, 0x74, - 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x42, 0x0a, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, - 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, - 0x52, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x4c, 0x0a, 0x0d, 0x73, 0x79, - 0x73, 0x74, 0x65, 0x6d, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x18, 0x18, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, - 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x41, 0x64, - 0x61, 0x70, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x73, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x12, 0x26, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x0e, 0x72, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, - 0x12, 0x49, 0x0a, 0x09, 0x48, 0x53, 0x4d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x1a, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x2b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x48, 0x77, 0x53, 0x65, 0x63, 0x75, - 0x72, 0x69, 0x74, 0x79, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x09, 0x48, 0x53, 0x4d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x48, - 0x53, 0x4d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x48, 0x53, - 0x4d, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x28, 0x0a, 0x0f, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x62, - 0x6f, 0x6f, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x18, 0x1c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, - 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x12, - 0x50, 0x0a, 0x11, 0x64, 0x61, 0x74, 0x61, 0x53, 0x65, 0x63, 0x41, 0x74, 0x52, 0x65, 0x73, 0x74, - 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, + 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x0b, 0x73, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x36, 0x0a, 0x08, 0x62, 0x6f, 0x6f, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, + 0x12, 0x37, 0x0a, 0x06, 0x73, 0x77, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x13, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, + 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, 0x53, + 0x57, 0x52, 0x06, 0x73, 0x77, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x48, 0x6f, 0x73, + 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x14, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x48, 0x6f, 0x73, + 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x4b, 0x0a, 0x0b, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, + 0x74, 0x65, 0x6d, 0x73, 0x18, 0x15, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, - 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x65, 0x63, 0x41, 0x74, 0x52, 0x65, 0x73, 0x74, 0x52, 0x11, - 0x64, 0x61, 0x74, 0x61, 0x53, 0x65, 0x63, 0x41, 0x74, 0x52, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, - 0x6f, 0x12, 0x3c, 0x0a, 0x08, 0x73, 0x65, 0x63, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x1e, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, - 0x74, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x65, 0x63, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x56, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x2e, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74, 0x72, 0x69, + 0x63, 0x49, 0x74, 0x65, 0x6d, 0x52, 0x0b, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, + 0x6d, 0x73, 0x12, 0x2a, 0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x62, 0x6f, 0x6f, 0x74, + 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x61, + 0x73, 0x74, 0x52, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x42, + 0x0a, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, + 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, + 0x6d, 0x65, 0x12, 0x4c, 0x0a, 0x0d, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x41, 0x64, 0x61, 0x70, + 0x74, 0x65, 0x72, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x49, 0x74, 0x65, 0x6d, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x49, 0x74, 0x65, - 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x49, 0x0a, 0x0c, 0x61, 0x70, 0x70, 0x49, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x20, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x41, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x52, 0x0c, 0x61, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x13, 0x72, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x43, 0x6f, 0x75, - 0x6e, 0x74, 0x65, 0x72, 0x12, 0x49, 0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x62, 0x6f, 0x6f, - 0x74, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, - 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x52, - 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x42, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, - 0x49, 0x0a, 0x0b, 0x63, 0x65, 0x6c, 0x6c, 0x5f, 0x72, 0x61, 0x64, 0x69, 0x6f, 0x73, 0x18, 0x23, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, - 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x43, 0x65, 0x6c, 0x6c, - 0x75, 0x6c, 0x61, 0x72, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, - 0x63, 0x65, 0x6c, 0x6c, 0x52, 0x61, 0x64, 0x69, 0x6f, 0x73, 0x12, 0x35, 0x0a, 0x04, 0x73, 0x69, - 0x6d, 0x73, 0x18, 0x24, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, - 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, - 0x53, 0x69, 0x6d, 0x63, 0x61, 0x72, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x73, 0x69, 0x6d, - 0x73, 0x12, 0x3b, 0x0a, 0x05, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x18, 0x25, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, 0x69, - 0x63, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x05, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x29, - 0x0a, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, - 0x64, 0x65, 0x18, 0x26, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, - 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x66, 0x0a, 0x17, 0x6d, 0x61, 0x69, - 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x5f, 0x72, 0x65, - 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x27, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, - 0x2e, 0x4d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, - 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x42, 0x02, 0x18, 0x01, 0x52, 0x15, 0x6d, 0x61, 0x69, 0x6e, - 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, - 0x6e, 0x12, 0x3a, 0x0a, 0x19, 0x68, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x5f, 0x77, 0x61, - 0x74, 0x63, 0x68, 0x64, 0x6f, 0x67, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x28, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x68, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x57, 0x61, - 0x74, 0x63, 0x68, 0x64, 0x6f, 0x67, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, - 0x11, 0x72, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x5f, 0x69, 0x6e, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x18, 0x29, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x65, 0x62, 0x6f, 0x6f, 0x74, - 0x49, 0x6e, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x45, 0x0a, 0x0c, 0x63, 0x61, - 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x2a, 0x20, 0x01, 0x28, 0x0b, + 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x0d, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, + 0x12, 0x26, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x65, 0x72, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x72, 0x65, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x49, 0x0a, 0x09, 0x48, 0x53, 0x4d, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2b, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x48, 0x77, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4d, 0x6f, 0x64, 0x75, + 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x09, 0x48, 0x53, 0x4d, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x48, 0x53, 0x4d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x1b, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x48, 0x53, 0x4d, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x28, 0x0a, + 0x0f, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, + 0x18, 0x1c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x62, 0x6f, + 0x6f, 0x74, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x12, 0x50, 0x0a, 0x11, 0x64, 0x61, 0x74, 0x61, 0x53, + 0x65, 0x63, 0x41, 0x74, 0x52, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x1d, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, + 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x65, 0x63, + 0x41, 0x74, 0x52, 0x65, 0x73, 0x74, 0x52, 0x11, 0x64, 0x61, 0x74, 0x61, 0x53, 0x65, 0x63, 0x41, + 0x74, 0x52, 0x65, 0x73, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3c, 0x0a, 0x08, 0x73, 0x65, 0x63, + 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, + 0x73, 0x65, 0x63, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x56, 0x0a, 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x1f, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, + 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x10, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x49, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x49, 0x0a, 0x0c, 0x61, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x18, + 0x20, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, + 0x6f, 0x41, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x0c, 0x61, 0x70, + 0x70, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x65, + 0x62, 0x6f, 0x6f, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, + 0x72, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x13, 0x72, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x49, 0x0a, 0x10, + 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x62, 0x6f, 0x6f, 0x74, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, + 0x18, 0x22, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x6f, 0x6f, + 0x74, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x42, 0x6f, 0x6f, + 0x74, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x49, 0x0a, 0x0b, 0x63, 0x65, 0x6c, 0x6c, 0x5f, + 0x72, 0x61, 0x64, 0x69, 0x6f, 0x73, 0x18, 0x23, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x5a, 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x4d, 0x6f, 0x64, 0x75, + 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x63, 0x65, 0x6c, 0x6c, 0x52, 0x61, 0x64, 0x69, + 0x6f, 0x73, 0x12, 0x35, 0x0a, 0x04, 0x73, 0x69, 0x6d, 0x73, 0x18, 0x24, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, - 0x69, 0x65, 0x73, 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, - 0x73, 0x12, 0x32, 0x0a, 0x15, 0x62, 0x61, 0x73, 0x65, 0x6f, 0x73, 0x5f, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x13, 0x62, 0x61, 0x73, 0x65, 0x6f, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x2c, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, - 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x44, 0x65, 0x76, 0x69, - 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x23, - 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x18, - 0x2d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x66, - 0x69, 0x6c, 0x65, 0x12, 0x64, 0x0a, 0x18, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, - 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x73, 0x18, - 0x2e, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, - 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x61, 0x69, 0x6e, - 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, - 0x6e, 0x52, 0x16, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, - 0x64, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x0a, 0x0c, 0x64, 0x6f, 0x72, - 0x6d, 0x61, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x2f, 0x20, 0x01, 0x28, 0x09, 0x42, - 0x02, 0x18, 0x01, 0x52, 0x0b, 0x64, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x12, 0x43, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, - 0x18, 0x30, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x74, 0x6f, - 0x72, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, - 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x36, 0x0a, 0x17, 0x73, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, - 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, - 0x18, 0x31, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, 0x73, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x4f, 0x0a, - 0x10, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, - 0x6f, 0x18, 0x32, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, - 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x74, - 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x61, - 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x49, - 0x0a, 0x0e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, - 0x18, 0x33, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x50, 0x49, - 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x0d, 0x61, 0x70, 0x69, 0x43, - 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x34, 0x0a, 0x16, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, - 0x6c, 0x65, 0x64, 0x18, 0x34, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x72, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, - 0x5e, 0x0a, 0x15, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x5f, 0x63, 0x61, 0x70, 0x61, - 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x35, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, - 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x43, 0x61, 0x70, - 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x14, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x61, 0x6c, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, - 0x1f, 0x0a, 0x0b, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x36, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x37, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x6e, 0x74, 0x65, 0x72, - 0x70, 0x72, 0x69, 0x73, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x6e, 0x74, - 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x38, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0c, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x49, 0x64, 0x12, 0x21, - 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x39, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, - 0x3a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, - 0x12, 0x44, 0x0a, 0x0e, 0x65, 0x6e, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x65, 0x72, - 0x74, 0x73, 0x18, 0x3c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x53, 0x69, 0x6d, 0x63, 0x61, 0x72, 0x64, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x73, 0x69, 0x6d, 0x73, 0x12, 0x3b, 0x0a, 0x05, 0x74, 0x61, 0x73, + 0x6b, 0x73, 0x18, 0x25, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, + 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, + 0x05, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, + 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x26, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0f, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, + 0x65, 0x12, 0x66, 0x0a, 0x17, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, + 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x27, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, + 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, + 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x42, 0x02, + 0x18, 0x01, 0x52, 0x15, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, + 0x6f, 0x64, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x3a, 0x0a, 0x19, 0x68, 0x61, 0x72, + 0x64, 0x77, 0x61, 0x72, 0x65, 0x5f, 0x77, 0x61, 0x74, 0x63, 0x68, 0x64, 0x6f, 0x67, 0x5f, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x28, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x68, 0x61, + 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x64, 0x6f, 0x67, 0x50, 0x72, + 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x72, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x5f, + 0x69, 0x6e, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x29, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x10, 0x72, 0x65, 0x62, 0x6f, 0x6f, 0x74, 0x49, 0x6e, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, + 0x73, 0x73, 0x12, 0x45, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, + 0x65, 0x73, 0x18, 0x2a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, - 0x65, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x65, 0x6e, 0x72, 0x6f, 0x6c, 0x6c, 0x65, - 0x64, 0x43, 0x65, 0x72, 0x74, 0x73, 0x22, 0x97, 0x01, 0x0a, 0x14, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x61, 0x6c, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, - 0x28, 0x0a, 0x10, 0x68, 0x76, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x76, - 0x69, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x68, 0x76, 0x54, 0x79, 0x70, - 0x65, 0x4b, 0x75, 0x62, 0x65, 0x76, 0x69, 0x72, 0x74, 0x12, 0x30, 0x0a, 0x14, 0x68, 0x77, 0x5f, - 0x69, 0x6e, 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x68, 0x77, 0x49, 0x6e, 0x76, 0x65, 0x6e, - 0x74, 0x6f, 0x72, 0x79, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, - 0x74, 0x63, 0x64, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0c, 0x65, 0x74, 0x63, 0x64, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x22, 0x84, 0x01, 0x0a, 0x0f, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x34, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x76, 0x0a, 0x11, 0x53, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x22, 0x0a, 0x0c, - 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x0c, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, - 0x12, 0x3d, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, - 0xd1, 0x02, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x3e, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, - 0x12, 0x3a, 0x0a, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x40, 0x0a, 0x0d, - 0x6c, 0x61, 0x73, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x12, 0x35, - 0x0a, 0x05, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, + 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x0c, 0x63, 0x61, 0x70, + 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x62, 0x61, 0x73, + 0x65, 0x6f, 0x73, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x65, 0x72, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x13, 0x62, 0x61, 0x73, 0x65, 0x6f, 0x73, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x37, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x2c, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x5a, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, + 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x2d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x64, 0x0a, 0x18, 0x6d, + 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x5f, + 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x73, 0x18, 0x2e, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x52, 0x05, - 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x6c, 0x61, 0x73, 0x74, 0x45, 0x72, 0x72, - 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x61, 0x73, 0x74, 0x45, 0x72, - 0x72, 0x6f, 0x72, 0x22, 0xeb, 0x08, 0x0a, 0x0a, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, - 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, - 0x0a, 0x06, 0x69, 0x73, 0x4d, 0x67, 0x6d, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, - 0x69, 0x73, 0x4d, 0x67, 0x6d, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x66, 0x72, 0x65, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x68, - 0x63, 0x70, 0x54, 0x79, 0x70, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x64, 0x68, - 0x63, 0x70, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, - 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x12, 0x18, - 0x0a, 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x74, 0x70, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x74, 0x70, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x6f, 0x72, 0x65, 0x5f, 0x6e, - 0x74, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x23, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0e, 0x6d, 0x6f, 0x72, 0x65, 0x4e, 0x74, 0x70, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, - 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x10, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, - 0x12, 0x22, 0x0a, 0x0c, 0x64, 0x68, 0x63, 0x70, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4c, 0x6f, 0x77, - 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x68, 0x63, 0x70, 0x52, 0x61, 0x6e, 0x67, - 0x65, 0x4c, 0x6f, 0x77, 0x12, 0x24, 0x0a, 0x0d, 0x64, 0x68, 0x63, 0x70, 0x52, 0x61, 0x6e, 0x67, - 0x65, 0x48, 0x69, 0x67, 0x68, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x64, 0x68, 0x63, - 0x70, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x48, 0x69, 0x67, 0x68, 0x12, 0x36, 0x0a, 0x05, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x18, 0x16, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x12, 0x18, 0x0a, 0x07, - 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x73, 0x18, 0x17, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x49, - 0x50, 0x41, 0x64, 0x64, 0x72, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, - 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x73, 0x18, 0x18, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, - 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, - 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6f, 0x72, - 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, - 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x4e, 0x53, 0x52, 0x03, 0x64, 0x6e, 0x73, 0x12, - 0x0e, 0x0a, 0x02, 0x75, 0x70, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x75, 0x70, 0x12, - 0x37, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x1b, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, - 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x47, 0x65, 0x6f, 0x4c, 0x6f, 0x63, 0x52, 0x08, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, - 0x1d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, - 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x03, 0x65, 0x72, 0x72, 0x12, 0x3d, 0x0a, 0x05, 0x75, 0x73, - 0x61, 0x67, 0x65, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, - 0x6e, 0x2e, 0x50, 0x68, 0x79, 0x49, 0x6f, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x55, 0x73, 0x61, - 0x67, 0x65, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x6e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x55, 0x55, 0x49, 0x44, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x55, 0x55, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x63, - 0x6f, 0x73, 0x74, 0x18, 0x20, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, 0x6f, 0x73, 0x74, 0x12, - 0x4c, 0x0a, 0x0f, 0x77, 0x69, 0x72, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, - 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x57, - 0x69, 0x72, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x77, - 0x69, 0x72, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, - 0x03, 0x6d, 0x74, 0x75, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x74, 0x75, 0x12, - 0x4c, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x18, 0x28, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, - 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, - 0x0c, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x40, 0x0a, - 0x0b, 0x70, 0x6e, 0x61, 0x63, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x32, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, - 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x50, 0x4e, 0x41, 0x43, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x0a, 0x70, 0x6e, 0x61, 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x40, 0x0a, 0x0b, 0x62, 0x6f, 0x6e, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x3c, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, - 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x6f, 0x6e, 0x64, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0a, 0x62, 0x6f, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x22, 0xf6, 0x01, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x39, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, - 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x69, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x0a, - 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, - 0x70, 0x61, 0x63, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, - 0x61, 0x63, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x12, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, 0x78, 0x79, - 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, 0x52, 0x4c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, 0x52, 0x4c, - 0x12, 0x18, 0x0a, 0x07, 0x77, 0x70, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x77, 0x70, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x22, 0x4c, 0x0a, 0x0a, 0x50, 0x72, - 0x6f, 0x78, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x89, 0x01, 0x0a, 0x0e, 0x57, 0x69, 0x72, - 0x65, 0x6c, 0x65, 0x73, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x35, 0x0a, 0x04, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x57, 0x69, 0x72, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x12, 0x40, 0x0a, 0x08, 0x63, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, - 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x43, 0x65, 0x6c, 0x6c, - 0x75, 0x6c, 0x61, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x08, 0x63, 0x65, 0x6c, 0x6c, - 0x75, 0x6c, 0x61, 0x72, 0x22, 0xf2, 0x03, 0x0a, 0x0f, 0x5a, 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, - 0x61, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x65, 0x6c, 0x6c, - 0x75, 0x6c, 0x61, 0x72, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0e, 0x63, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x4d, 0x6f, 0x64, 0x75, 0x6c, - 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x69, 0x6d, 0x5f, 0x63, 0x61, 0x72, 0x64, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x73, 0x69, 0x6d, 0x43, 0x61, 0x72, 0x64, 0x73, 0x12, 0x44, - 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, - 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, - 0x72, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x76, 0x69, - 0x64, 0x65, 0x72, 0x73, 0x12, 0x4f, 0x0a, 0x0c, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, - 0x72, 0x61, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, - 0x6f, 0x6e, 0x2e, 0x52, 0x61, 0x64, 0x69, 0x6f, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x65, - 0x63, 0x68, 0x6e, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x52, 0x0b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, - 0x74, 0x52, 0x61, 0x74, 0x73, 0x12, 0x3d, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x65, 0x64, 0x41, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x6f, 0x62, 0x65, - 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, - 0x6f, 0x62, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x3d, 0x0a, 0x07, 0x62, 0x65, 0x61, 0x72, - 0x65, 0x72, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x42, 0x65, 0x61, 0x72, 0x65, 0x72, 0x52, 0x07, - 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x73, 0x12, 0x40, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x66, 0x69, - 0x6c, 0x65, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, - 0x08, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x22, 0xdc, 0x04, 0x0a, 0x0a, 0x5a, 0x49, - 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, 0x53, 0x57, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x63, 0x74, 0x69, - 0x76, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x63, 0x74, - 0x69, 0x76, 0x61, 0x74, 0x65, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x28, - 0x0a, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x76, 0x69, 0x63, - 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x74, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x35, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, + 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, + 0x6f, 0x64, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x52, 0x16, 0x6d, 0x61, 0x69, 0x6e, 0x74, + 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, + 0x73, 0x12, 0x25, 0x0a, 0x0c, 0x64, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x2f, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0b, 0x64, 0x6f, 0x72, + 0x6d, 0x61, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x43, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, + 0x61, 0x67, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x30, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x36, 0x0a, + 0x17, 0x73, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x31, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x15, + 0x73, 0x68, 0x75, 0x74, 0x64, 0x6f, 0x77, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x4f, 0x0a, 0x10, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x32, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, + 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0f, 0x61, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x49, 0x0a, 0x0e, 0x61, 0x70, 0x69, 0x5f, 0x63, 0x61, + 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x33, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x50, 0x49, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, + 0x74, 0x79, 0x52, 0x0d, 0x61, 0x70, 0x69, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, + 0x79, 0x12, 0x34, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x61, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x5f, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x34, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x14, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x44, + 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x5e, 0x0a, 0x15, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x61, 0x6c, 0x5f, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, + 0x18, 0x35, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, + 0x73, 0x52, 0x14, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x43, 0x61, 0x70, 0x61, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x65, 0x76, 0x69, 0x63, + 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x36, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x65, + 0x76, 0x69, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6e, 0x74, 0x65, + 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x37, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0e, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x38, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6e, 0x74, 0x65, 0x72, 0x70, + 0x72, 0x69, 0x73, 0x65, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x39, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x3a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x44, 0x0a, 0x0e, 0x65, 0x6e, 0x72, 0x6f, + 0x6c, 0x6c, 0x65, 0x64, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x73, 0x18, 0x3c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, - 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x68, 0x6f, 0x72, 0x74, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, - 0x68, 0x6f, 0x72, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x6c, - 0x6f, 0x6e, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x6c, 0x6f, 0x6e, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x34, 0x0a, - 0x05, 0x73, 0x77, 0x45, 0x72, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, - 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, - 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x73, 0x77, - 0x45, 0x72, 0x72, 0x12, 0x2a, 0x0a, 0x10, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, - 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x64, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, - 0x41, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x0b, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x73, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, - 0x74, 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x75, 0x62, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x53, 0x74, 0x72, 0x12, 0x42, 0x0a, 0x09, 0x73, 0x75, 0x62, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x42, 0x61, 0x73, 0x65, 0x4f, 0x73, 0x53, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x09, 0x73, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x73, 0x75, - 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, - 0x0e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x73, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x80, 0x02, 0x0a, 0x0c, 0x5a, 0x49, 0x6e, - 0x66, 0x6f, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x65, 0x76, - 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x65, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x61, 0x74, 0x68, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, - 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, - 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x28, 0x0a, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, - 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x26, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x61, 0x62, 0x65, - 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x2c, 0x0a, 0x11, 0x70, 0x61, 0x72, 0x74, 0x69, - 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x47, 0x75, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x11, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, - 0x65, 0x47, 0x75, 0x69, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, - 0x6f, 0x6e, 0x55, 0x75, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x61, - 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x75, 0x69, 0x64, 0x22, 0x92, 0x02, 0x0a, 0x0d, - 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1b, 0x0a, - 0x09, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x3b, 0x0a, 0x0b, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x65, 0x72, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x0d, 0x65, 0x6e, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x64, 0x43, 0x65, 0x72, 0x74, 0x73, 0x12, 0x44, + 0x0a, 0x09, 0x63, 0x70, 0x75, 0x5f, 0x70, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x3d, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, + 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x50, 0x55, 0x50, 0x6f, 0x6f, 0x6c, 0x55, + 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x63, 0x70, 0x75, 0x50, + 0x6f, 0x6f, 0x6c, 0x73, 0x22, 0xcb, 0x01, 0x0a, 0x14, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, + 0x6c, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x28, 0x0a, + 0x10, 0x68, 0x76, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6b, 0x75, 0x62, 0x65, 0x76, 0x69, 0x72, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x68, 0x76, 0x54, 0x79, 0x70, 0x65, 0x4b, + 0x75, 0x62, 0x65, 0x76, 0x69, 0x72, 0x74, 0x12, 0x30, 0x0a, 0x14, 0x68, 0x77, 0x5f, 0x69, 0x6e, + 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x72, 0x79, 0x5f, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x68, 0x77, 0x49, 0x6e, 0x76, 0x65, 0x6e, 0x74, 0x6f, + 0x72, 0x79, 0x53, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x74, 0x63, + 0x64, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0c, 0x65, 0x74, 0x63, 0x64, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x32, + 0x0a, 0x15, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x64, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x69, 0x73, + 0x6f, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x64, 0x43, 0x70, 0x75, 0x49, 0x73, 0x6f, 0x6c, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x22, 0x84, 0x01, 0x0a, 0x0f, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x74, 0x74, 0x65, + 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x34, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, + 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x76, 0x0a, 0x11, 0x53, 0x79, 0x73, + 0x74, 0x65, 0x6d, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x22, + 0x0a, 0x0c, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x64, + 0x65, 0x78, 0x12, 0x3d, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, + 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x50, + 0x6f, 0x72, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x22, 0xd1, 0x02, 0x0a, 0x10, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x3e, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, + 0x74, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, + 0x74, 0x79, 0x12, 0x3a, 0x0a, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x35, - 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6f, - 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, - 0x66, 0x6f, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x5f, 0x65, 0x72, - 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, + 0x6d, 0x70, 0x52, 0x0a, 0x6c, 0x61, 0x73, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x40, + 0x0a, 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, + 0x12, 0x35, 0x0a, 0x05, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, + 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, + 0x52, 0x05, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x6c, 0x61, 0x73, 0x74, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6c, 0x61, 0x73, 0x74, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xeb, 0x08, 0x0a, 0x0a, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, + 0x50, 0x6f, 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x66, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x16, 0x0a, 0x06, 0x69, 0x73, 0x4d, 0x67, 0x6d, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x06, 0x69, 0x73, 0x4d, 0x67, 0x6d, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x65, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x66, 0x72, 0x65, 0x65, 0x12, 0x1a, 0x0a, 0x08, + 0x64, 0x68, 0x63, 0x70, 0x54, 0x79, 0x70, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, + 0x64, 0x68, 0x63, 0x70, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x75, 0x62, 0x6e, + 0x65, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, + 0x12, 0x18, 0x0a, 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x0d, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x6f, + 0x6d, 0x61, 0x69, 0x6e, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x74, + 0x70, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, + 0x74, 0x70, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x6f, 0x72, 0x65, + 0x5f, 0x6e, 0x74, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x23, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0e, 0x6d, 0x6f, 0x72, 0x65, 0x4e, 0x74, 0x70, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, + 0x18, 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x64, 0x68, 0x63, 0x70, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4c, + 0x6f, 0x77, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x68, 0x63, 0x70, 0x52, 0x61, + 0x6e, 0x67, 0x65, 0x4c, 0x6f, 0x77, 0x12, 0x24, 0x0a, 0x0d, 0x64, 0x68, 0x63, 0x70, 0x52, 0x61, + 0x6e, 0x67, 0x65, 0x48, 0x69, 0x67, 0x68, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x64, + 0x68, 0x63, 0x70, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x48, 0x69, 0x67, 0x68, 0x12, 0x36, 0x0a, 0x05, + 0x70, 0x72, 0x6f, 0x78, 0x79, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x18, + 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x41, 0x64, 0x64, 0x72, 0x12, 0x18, + 0x0a, 0x07, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x73, 0x18, 0x17, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x07, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x64, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x73, 0x18, 0x18, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x73, + 0x12, 0x2f, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, + 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x4e, 0x53, 0x52, 0x03, 0x64, 0x6e, + 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x75, 0x70, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x75, + 0x70, 0x12, 0x37, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x1b, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x47, 0x65, 0x6f, 0x4c, 0x6f, 0x63, + 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x03, 0x65, 0x72, + 0x72, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, - 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x6e, 0x61, 0x70, 0x45, 0x72, 0x72, - 0x22, 0x60, 0x0a, 0x10, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0a, 0x6e, 0x6f, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x22, 0x86, 0x05, 0x0a, 0x08, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x41, 0x70, 0x70, 0x12, - 0x14, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x41, 0x70, 0x70, 0x49, 0x44, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x70, 0x70, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x70, 0x70, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x41, - 0x70, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, - 0x41, 0x70, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x41, 0x70, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x41, 0x70, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x40, 0x0a, - 0x0c, 0x73, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x08, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x53, - 0x57, 0x52, 0x0c, 0x73, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, - 0x36, 0x0a, 0x08, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x62, - 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x4a, 0x0a, 0x10, 0x61, 0x73, 0x73, 0x69, 0x67, - 0x6e, 0x65, 0x64, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, - 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x69, 0x6f, 0x42, 0x75, 0x6e, 0x64, 0x6c, - 0x65, 0x52, 0x10, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x41, 0x64, 0x61, 0x70, 0x74, - 0x65, 0x72, 0x73, 0x12, 0x36, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x45, 0x72, 0x72, 0x18, 0x0e, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x61, 0x70, 0x70, 0x45, 0x72, 0x72, 0x12, 0x33, 0x0a, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, + 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x03, 0x65, 0x72, 0x72, 0x12, 0x3d, 0x0a, 0x05, + 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, 0x6d, + 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x68, 0x79, 0x49, 0x6f, 0x4d, 0x65, 0x6d, 0x62, 0x65, 0x72, 0x55, + 0x73, 0x61, 0x67, 0x65, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x6e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x55, 0x55, 0x49, 0x44, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x55, 0x55, 0x49, 0x44, 0x12, 0x12, 0x0a, + 0x04, 0x63, 0x6f, 0x73, 0x74, 0x18, 0x20, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, 0x6f, 0x73, + 0x74, 0x12, 0x4c, 0x0a, 0x0f, 0x77, 0x69, 0x72, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x5f, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x18, 0x21, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, - 0x2e, 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x3b, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x10, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, - 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1e, 0x0a, - 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x66, 0x73, 0x18, 0x11, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x66, 0x73, 0x12, 0x40, 0x0a, - 0x09, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x18, 0x12, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, - 0x73, 0x68, 0x6f, 0x74, 0x52, 0x09, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x12, - 0x2e, 0x0a, 0x13, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x70, 0x5f, 0x72, - 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x18, 0x14, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x70, 0x70, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x4a, - 0x04, 0x08, 0x09, 0x10, 0x0c, 0x4a, 0x04, 0x08, 0x13, 0x10, 0x14, 0x22, 0x5e, 0x0a, 0x10, 0x5a, - 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x14, 0x0a, 0x05, 0x73, 0x70, 0x69, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x73, 0x70, 0x69, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x75, 0x62, 0x4e, 0x65, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x75, 0x62, 0x4e, 0x65, 0x74, 0x12, 0x1c, 0x0a, - 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xb2, 0x02, 0x0a, 0x0c, - 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x71, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x72, 0x65, 0x71, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x69, 0x6e, 0x73, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x73, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x65, 0x73, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x38, 0x0a, 0x05, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6f, 0x72, + 0x2e, 0x57, 0x69, 0x72, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x0e, 0x77, 0x69, 0x72, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x10, 0x0a, 0x03, 0x6d, 0x74, 0x75, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x74, + 0x75, 0x12, 0x4c, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x18, 0x28, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, + 0x2e, 0x50, 0x6f, 0x72, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, + 0x40, 0x0a, 0x0b, 0x70, 0x6e, 0x61, 0x63, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x32, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x50, 0x4e, 0x41, 0x43, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0a, 0x70, 0x6e, 0x61, 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x40, 0x0a, 0x0b, 0x62, 0x6f, 0x6e, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x3c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x6f, 0x6e, + 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0a, 0x62, 0x6f, 0x6e, 0x64, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x22, 0xf6, 0x01, 0x0a, 0x0b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x39, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x69, 0x65, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x78, 0x69, 0x65, 0x73, 0x12, 0x1e, + 0x0a, 0x0a, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x70, 0x61, 0x63, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x70, 0x61, 0x63, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x6e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, + 0x78, 0x79, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x6e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, 0x52, 0x4c, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x55, + 0x52, 0x4c, 0x12, 0x18, 0x0a, 0x07, 0x77, 0x70, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x77, 0x70, 0x61, 0x64, 0x55, 0x52, 0x4c, 0x22, 0x4c, 0x0a, 0x0a, + 0x50, 0x72, 0x6f, 0x78, 0x79, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, + 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x89, 0x01, 0x0a, 0x0e, 0x57, + 0x69, 0x72, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x35, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x12, 0x40, 0x0a, 0x08, 0x63, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x43, 0x65, + 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x08, 0x63, 0x65, + 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x22, 0xf2, 0x03, 0x0a, 0x0f, 0x5a, 0x43, 0x65, 0x6c, 0x6c, + 0x75, 0x6c, 0x61, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x65, + 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x4d, 0x6f, 0x64, + 0x75, 0x6c, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x69, 0x6d, 0x5f, 0x63, 0x61, 0x72, 0x64, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x73, 0x69, 0x6d, 0x43, 0x61, 0x72, 0x64, 0x73, + 0x12, 0x44, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x43, 0x65, 0x6c, 0x6c, 0x75, + 0x6c, 0x61, 0x72, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x52, 0x09, 0x70, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x12, 0x4f, 0x0a, 0x0c, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, + 0x74, 0x5f, 0x72, 0x61, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x63, 0x6f, + 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x61, 0x64, 0x69, 0x6f, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x54, 0x65, 0x63, 0x68, 0x6e, 0x6f, 0x6c, 0x6f, 0x67, 0x79, 0x52, 0x0b, 0x63, 0x75, 0x72, 0x72, + 0x65, 0x6e, 0x74, 0x52, 0x61, 0x74, 0x73, 0x12, 0x3d, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x72, 0x6f, + 0x62, 0x65, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x70, 0x72, 0x6f, 0x62, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x3d, 0x0a, 0x07, 0x62, 0x65, + 0x61, 0x72, 0x65, 0x72, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x42, 0x65, 0x61, 0x72, 0x65, 0x72, + 0x52, 0x07, 0x62, 0x65, 0x61, 0x72, 0x65, 0x72, 0x73, 0x12, 0x40, 0x0a, 0x08, 0x70, 0x72, 0x6f, + 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, + 0x65, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x22, 0xdc, 0x04, 0x0a, 0x0a, + 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, 0x53, 0x57, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x63, + 0x74, 0x69, 0x76, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, + 0x63, 0x74, 0x69, 0x76, 0x61, 0x74, 0x65, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x74, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x12, 0x28, 0x0a, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x76, + 0x69, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x70, 0x61, + 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x35, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, + 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x68, 0x6f, + 0x72, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0c, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, + 0x0b, 0x6c, 0x6f, 0x6e, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x6c, 0x6f, 0x6e, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x34, 0x0a, 0x05, 0x73, 0x77, 0x45, 0x72, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, + 0x73, 0x77, 0x45, 0x72, 0x72, 0x12, 0x2a, 0x0a, 0x10, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, + 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x10, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x12, 0x41, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, + 0x0b, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x42, 0x61, 0x73, 0x65, + 0x4f, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x73, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x53, 0x74, 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x75, 0x62, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x53, 0x74, 0x72, 0x12, 0x42, 0x0a, 0x09, 0x73, 0x75, 0x62, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x73, 0x53, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x09, 0x73, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2c, 0x0a, 0x11, + 0x73, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x11, 0x73, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x22, 0x80, 0x02, 0x0a, 0x0c, 0x5a, + 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, + 0x65, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x65, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x61, 0x74, 0x68, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x75, 0x6e, 0x74, 0x50, 0x61, 0x74, + 0x68, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x28, 0x0a, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x26, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x61, + 0x62, 0x65, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x2c, 0x0a, 0x11, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x47, 0x75, 0x69, 0x64, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x54, + 0x79, 0x70, 0x65, 0x47, 0x75, 0x69, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x75, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x75, 0x69, 0x64, 0x22, 0x92, 0x02, + 0x0a, 0x0d, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x3b, 0x0a, 0x0b, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, + 0x12, 0x35, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x5f, + 0x65, 0x72, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x6e, 0x61, 0x70, 0x45, + 0x72, 0x72, 0x22, 0x60, 0x0a, 0x10, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x4c, 0x0a, 0x0b, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2b, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, - 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, - 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x18, - 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, + 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0a, 0x6e, 0x6f, 0x64, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x22, 0x86, 0x05, 0x0a, 0x08, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x41, 0x70, + 0x70, 0x12, 0x14, 0x0a, 0x05, 0x41, 0x70, 0x70, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x41, 0x70, 0x70, 0x49, 0x44, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x70, 0x70, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x70, 0x70, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x41, 0x70, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x73, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x41, 0x70, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x41, 0x70, 0x70, 0x4e, 0x61, 0x6d, 0x65, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x41, 0x70, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x40, 0x0a, 0x0c, 0x73, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, + 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, - 0x6f, 0x56, 0x70, 0x6e, 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x6c, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, 0x05, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x0b, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, + 0x6f, 0x53, 0x57, 0x52, 0x0c, 0x73, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x36, 0x0a, 0x08, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x0c, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x08, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x4a, 0x0a, 0x10, 0x61, 0x73, 0x73, + 0x69, 0x67, 0x6e, 0x65, 0x64, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x73, 0x18, 0x0d, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x69, 0x6f, 0x42, 0x75, 0x6e, + 0x64, 0x6c, 0x65, 0x52, 0x10, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x41, 0x64, 0x61, + 0x70, 0x74, 0x65, 0x72, 0x73, 0x12, 0x36, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x45, 0x72, 0x72, 0x18, + 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, + 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x61, 0x70, 0x70, 0x45, 0x72, 0x72, 0x12, 0x33, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x3b, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x10, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, + 0x1e, 0x0a, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x66, 0x73, 0x18, 0x11, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x66, 0x73, 0x12, + 0x40, 0x0a, 0x09, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x73, 0x18, 0x12, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, + 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x6e, + 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x09, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x61, 0x70, 0x70, + 0x5f, 0x72, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x18, 0x14, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, + 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x41, 0x70, 0x70, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, + 0x67, 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0c, 0x4a, 0x04, 0x08, 0x13, 0x10, 0x14, 0x22, 0x5e, 0x0a, + 0x10, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x70, 0x69, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x73, 0x70, 0x69, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x75, 0x62, 0x4e, 0x65, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x75, 0x62, 0x4e, 0x65, 0x74, 0x12, + 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xb2, 0x02, + 0x0a, 0x0c, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x71, 0x49, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x72, 0x65, 0x71, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x69, 0x6e, 0x73, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x73, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x65, 0x73, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x38, + 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, + 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, + 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x6c, 0x49, 0x6e, 0x66, + 0x6f, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, + 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, + 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, + 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, 0x05, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x0b, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, + 0x56, 0x70, 0x6e, 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x72, 0x49, 0x6e, + 0x66, 0x6f, 0x22, 0x4e, 0x0a, 0x10, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x45, 0x6e, + 0x64, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x12, 0x12, + 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, + 0x72, 0x74, 0x22, 0xe7, 0x02, 0x0a, 0x0c, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x43, + 0x6f, 0x6e, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x07, 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, + 0x6b, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, 0x6b, 0x65, 0x73, 0x12, + 0x38, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x6c, 0x49, 0x6e, + 0x66, 0x6f, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, + 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x45, 0x6e, 0x64, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, + 0x05, 0x6c, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, 0x05, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, + 0x6f, 0x56, 0x70, 0x6e, 0x45, 0x6e, 0x64, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x05, 0x72, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x0a, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, - 0x6e, 0x4c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x72, 0x49, 0x6e, 0x66, 0x6f, - 0x22, 0x4e, 0x0a, 0x10, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x45, 0x6e, 0x64, 0x50, - 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x12, 0x12, 0x0a, 0x04, - 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, - 0x22, 0xe7, 0x02, 0x0a, 0x0c, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x43, 0x6f, 0x6e, - 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, - 0x18, 0x0a, 0x07, 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x07, 0x65, 0x73, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x6b, 0x65, - 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, 0x6b, 0x65, 0x73, 0x12, 0x38, 0x0a, - 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6f, + 0x6e, 0x4c, 0x69, 0x6e, 0x6b, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, 0xa7, 0x01, 0x0a, + 0x08, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x54, + 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x75, 0x70, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x42, 0x61, 0x73, 0x65, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x42, 0x61, + 0x73, 0x65, 0x64, 0x12, 0x2a, 0x0a, 0x10, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x69, 0x6e, 0x67, + 0x49, 0x70, 0x41, 0x64, 0x64, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x6c, + 0x69, 0x73, 0x74, 0x65, 0x6e, 0x69, 0x6e, 0x67, 0x49, 0x70, 0x41, 0x64, 0x64, 0x72, 0x73, 0x12, + 0x35, 0x0a, 0x04, 0x63, 0x6f, 0x6e, 0x6e, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, + 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, + 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, + 0x52, 0x04, 0x63, 0x6f, 0x6e, 0x6e, 0x22, 0x9f, 0x08, 0x0a, 0x14, 0x5a, 0x49, 0x6e, 0x66, 0x6f, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, + 0x1c, 0x0a, 0x09, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x44, 0x12, 0x26, 0x0a, + 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x69, 0x6e, 0x73, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x63, 0x74, 0x69, 0x76, 0x61, 0x74, 0x65, 0x64, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x63, 0x74, 0x69, 0x76, 0x61, 0x74, 0x65, + 0x64, 0x12, 0x3c, 0x0a, 0x0b, 0x75, 0x70, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x0b, 0x75, 0x70, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70, 0x12, + 0x40, 0x0a, 0x0c, 0x73, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, + 0x6f, 0x53, 0x57, 0x52, 0x0c, 0x73, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x30, 0x0a, 0x11, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x6c, 0x69, + 0x6e, 0x6b, 0x49, 0x6e, 0x74, 0x66, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, + 0x52, 0x11, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x6c, 0x69, 0x6e, 0x6b, 0x49, + 0x6e, 0x74, 0x66, 0x12, 0x32, 0x0a, 0x12, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x55, 0x70, + 0x6c, 0x69, 0x6e, 0x6b, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x42, + 0x02, 0x18, 0x01, 0x52, 0x12, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x6c, 0x69, + 0x6e, 0x6b, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x6f, 0x72, 0x74, 0x73, + 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x1c, 0x0a, + 0x09, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x4e, 0x75, 0x6d, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x09, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x4e, 0x75, 0x6d, 0x12, 0x1e, 0x0a, 0x0a, 0x62, + 0x72, 0x69, 0x64, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x15, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x62, + 0x72, 0x69, 0x64, 0x67, 0x65, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x18, 0x16, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0c, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x12, + 0x50, 0x0a, 0x0d, 0x69, 0x70, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, + 0x18, 0x17, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x6d, 0x65, + 0x74, 0x49, 0x50, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0d, 0x69, 0x70, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x73, 0x12, 0x34, 0x0a, 0x04, 0x76, 0x69, 0x66, 0x73, 0x18, 0x19, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, + 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x6d, 0x65, 0x74, 0x56, 0x69, 0x66, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x04, 0x76, 0x69, 0x66, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x69, 0x70, 0x76, 0x34, 0x45, + 0x69, 0x64, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x70, 0x76, 0x34, 0x45, 0x69, + 0x64, 0x12, 0x4a, 0x0a, 0x10, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x41, 0x64, 0x61, + 0x70, 0x74, 0x65, 0x72, 0x73, 0x18, 0x1e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x5a, 0x69, 0x6f, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x10, 0x61, 0x73, 0x73, + 0x69, 0x67, 0x6e, 0x65, 0x64, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x73, 0x12, 0x35, 0x0a, + 0x05, 0x76, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, - 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x6c, 0x49, 0x6e, 0x66, 0x6f, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, - 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x45, 0x6e, 0x64, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x05, 0x6c, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3b, 0x0a, 0x05, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, - 0x70, 0x6e, 0x45, 0x6e, 0x64, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x05, 0x72, 0x49, 0x6e, 0x66, - 0x6f, 0x12, 0x37, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x4c, - 0x69, 0x6e, 0x6b, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, 0xa7, 0x01, 0x0a, 0x08, 0x5a, - 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x54, 0x69, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x75, 0x70, 0x54, 0x69, 0x6d, 0x65, 0x12, - 0x20, 0x0a, 0x0b, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x42, 0x61, 0x73, 0x65, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x42, 0x61, 0x73, 0x65, - 0x64, 0x12, 0x2a, 0x0a, 0x10, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x69, 0x6e, 0x67, 0x49, 0x70, - 0x41, 0x64, 0x64, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x69, 0x6e, 0x67, 0x49, 0x70, 0x41, 0x64, 0x64, 0x72, 0x73, 0x12, 0x35, 0x0a, - 0x04, 0x63, 0x6f, 0x6e, 0x6e, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, + 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x48, 0x00, 0x52, 0x05, 0x76, + 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x3e, 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x45, + 0x72, 0x72, 0x18, 0x28, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, + 0x6b, 0x45, 0x72, 0x72, 0x12, 0x40, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x29, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x74, 0x75, 0x18, 0x2a, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x74, 0x75, 0x12, 0x39, 0x0a, 0x09, 0x69, 0x70, 0x5f, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x2b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, - 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x43, 0x6f, 0x6e, 0x6e, 0x52, 0x04, - 0x63, 0x6f, 0x6e, 0x6e, 0x22, 0x9f, 0x08, 0x0a, 0x14, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x65, - 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x1c, 0x0a, - 0x09, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x44, 0x12, 0x26, 0x0a, 0x0e, 0x6e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x69, 0x6e, 0x73, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x20, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x63, 0x74, 0x69, 0x76, 0x61, 0x74, 0x65, 0x64, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x63, 0x74, 0x69, 0x76, 0x61, 0x74, 0x65, 0x64, 0x12, - 0x3c, 0x0a, 0x0b, 0x75, 0x70, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x0b, 0x75, 0x70, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x40, 0x0a, - 0x0c, 0x73, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x09, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x53, - 0x57, 0x52, 0x0c, 0x73, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, - 0x30, 0x0a, 0x11, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x6c, 0x69, 0x6e, 0x6b, - 0x49, 0x6e, 0x74, 0x66, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x11, - 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x6c, 0x69, 0x6e, 0x6b, 0x49, 0x6e, 0x74, - 0x66, 0x12, 0x32, 0x0a, 0x12, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x6c, 0x69, - 0x6e, 0x6b, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, - 0x01, 0x52, 0x12, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x6c, 0x69, 0x6e, 0x6b, - 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x0c, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x62, - 0x72, 0x69, 0x64, 0x67, 0x65, 0x4e, 0x75, 0x6d, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, - 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x4e, 0x75, 0x6d, 0x12, 0x1e, 0x0a, 0x0a, 0x62, 0x72, 0x69, - 0x64, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x15, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x62, - 0x72, 0x69, 0x64, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x62, 0x72, 0x69, - 0x64, 0x67, 0x65, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x18, 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x62, 0x72, 0x69, 0x64, 0x67, 0x65, 0x49, 0x50, 0x41, 0x64, 0x64, 0x72, 0x12, 0x50, 0x0a, - 0x0d, 0x69, 0x70, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x17, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, - 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x6d, 0x65, 0x74, 0x49, - 0x50, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x0d, 0x69, 0x70, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, - 0x34, 0x0a, 0x04, 0x76, 0x69, 0x66, 0x73, 0x18, 0x19, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x6d, 0x65, 0x74, 0x56, 0x69, 0x66, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x04, 0x76, 0x69, 0x66, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x69, 0x70, 0x76, 0x34, 0x45, 0x69, 0x64, - 0x18, 0x1a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x70, 0x76, 0x34, 0x45, 0x69, 0x64, 0x12, - 0x4a, 0x0a, 0x10, 0x61, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x41, 0x64, 0x61, 0x70, 0x74, - 0x65, 0x72, 0x73, 0x18, 0x1e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x5a, 0x69, 0x6f, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x10, 0x61, 0x73, 0x73, 0x69, 0x67, - 0x6e, 0x65, 0x64, 0x41, 0x64, 0x61, 0x70, 0x74, 0x65, 0x72, 0x73, 0x12, 0x35, 0x0a, 0x05, 0x76, - 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, - 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x48, 0x00, 0x52, 0x05, 0x76, 0x69, 0x6e, - 0x66, 0x6f, 0x12, 0x3e, 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x45, 0x72, 0x72, - 0x18, 0x28, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, - 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x45, - 0x72, 0x72, 0x12, 0x40, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x29, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x2a, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, - 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, - 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x74, 0x75, 0x18, 0x2a, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x03, 0x6d, 0x74, 0x75, 0x12, 0x39, 0x0a, 0x09, 0x69, 0x70, 0x5f, 0x72, 0x6f, 0x75, - 0x74, 0x65, 0x73, 0x18, 0x2b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x49, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x08, 0x69, 0x70, 0x52, 0x6f, 0x75, 0x74, 0x65, - 0x73, 0x42, 0x0d, 0x0a, 0x0b, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x4a, 0x04, 0x08, 0x18, 0x10, 0x19, 0x22, 0x89, 0x01, 0x0a, 0x07, 0x49, 0x50, 0x52, 0x6f, 0x75, - 0x74, 0x65, 0x12, 0x2f, 0x0a, 0x13, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x12, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x12, 0x18, 0x0a, 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x12, 0x0a, - 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, - 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x5f, 0x61, 0x70, 0x70, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x41, - 0x70, 0x70, 0x22, 0xb7, 0x01, 0x0a, 0x09, 0x55, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x12, 0x3a, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, - 0x72, 0x65, 0x66, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, - 0x72, 0x65, 0x66, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, - 0x52, 0x65, 0x66, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x54, 0x69, - 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x66, 0x63, 0x6f, 0x75, - 0x6e, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x59, 0x0a, 0x0f, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, - 0x22, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, - 0x74, 0x65, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x75, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, - 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x63, 0x75, 0x72, 0x53, 0x69, - 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x8b, 0x03, 0x0a, 0x0b, 0x5a, 0x49, 0x6e, 0x66, - 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x64, - 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, - 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, + 0x6f, 0x2e, 0x49, 0x50, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x08, 0x69, 0x70, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x73, 0x42, 0x0d, 0x0a, 0x0b, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x4a, 0x04, 0x08, 0x18, 0x10, 0x19, 0x22, 0x89, 0x01, 0x0a, 0x07, 0x49, 0x50, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x12, 0x2f, 0x0a, 0x13, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x12, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x18, 0x0a, 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, + 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, + 0x6f, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x5f, 0x61, + 0x70, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x41, 0x70, 0x70, 0x22, 0xb7, 0x01, 0x0a, 0x09, 0x55, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x3a, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1a, + 0x0a, 0x08, 0x72, 0x65, 0x66, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x08, 0x72, 0x65, 0x66, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, + 0x73, 0x74, 0x52, 0x65, 0x66, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, + 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x66, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x59, + 0x0a, 0x0f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, + 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x75, 0x72, 0x53, 0x69, 0x7a, 0x65, + 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x63, 0x75, 0x72, + 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x8b, 0x03, 0x0a, 0x0b, 0x5a, 0x49, + 0x6e, 0x66, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x20, 0x0a, + 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x34, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, + 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x42, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x09, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, + 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2e, + 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, + 0x74, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x12, 0x3c, + 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x72, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, + 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x72, 0x72, 0x12, 0x29, 0x0a, 0x10, + 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x36, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x63, + 0x75, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x0c, 0x63, 0x75, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, + 0xc9, 0x03, 0x0a, 0x10, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x54, 0x72, 0x65, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, + 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, + 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, + 0x61, 0x32, 0x35, 0x36, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x68, 0x61, 0x32, + 0x35, 0x36, 0x12, 0x43, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x09, 0x72, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x34, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x55, 0x73, 0x61, + 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x33, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, - 0x66, 0x6f, 0x2e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x75, 0x73, - 0x61, 0x67, 0x65, 0x12, 0x42, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x09, 0x72, 0x65, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, - 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x53, 0x77, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2e, 0x0a, 0x12, + 0x66, 0x6f, 0x2e, 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, + 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, - 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, - 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x12, 0x3c, 0x0a, 0x09, - 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x72, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x67, 0x65, 0x12, 0x30, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x45, 0x72, 0x72, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x36, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x75, 0x72, - 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x0c, 0x63, 0x75, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0xc9, 0x03, - 0x0a, 0x10, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x72, - 0x65, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, - 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x69, 0x73, - 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x32, - 0x35, 0x36, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, - 0x12, 0x43, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x34, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, - 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x55, 0x73, 0x61, 0x67, 0x65, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x33, 0x0a, 0x05, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, - 0x2e, 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x2e, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, - 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x70, 0x72, - 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, - 0x12, 0x30, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x03, 0x65, - 0x72, 0x72, 0x12, 0x2a, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, - 0x68, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, - 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x68, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x29, - 0x0a, 0x10, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, - 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xb5, 0x02, 0x0a, 0x09, 0x5a, 0x49, - 0x6e, 0x66, 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, - 0x36, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x12, - 0x43, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, - 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x73, 0x12, 0x34, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x33, 0x0a, 0x05, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, - 0x2e, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, - 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x70, 0x72, 0x6f, - 0x67, 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x12, - 0x30, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, - 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, - 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x03, 0x65, 0x72, - 0x72, 0x22, 0x43, 0x0a, 0x0d, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x4c, 0x69, - 0x73, 0x74, 0x12, 0x32, 0x0a, 0x04, 0x62, 0x6c, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, + 0x03, 0x65, 0x72, 0x72, 0x12, 0x2a, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, + 0x74, 0x53, 0x68, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, + 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x68, 0x61, 0x4c, 0x69, 0x73, 0x74, + 0x12, 0x29, 0x0a, 0x10, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x67, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xb5, 0x02, 0x0a, 0x09, + 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x68, 0x61, + 0x32, 0x35, 0x36, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x68, 0x61, 0x32, 0x35, + 0x36, 0x12, 0x43, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x52, 0x09, 0x72, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x34, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x55, 0x73, 0x61, 0x67, + 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x12, 0x33, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, 0x72, + 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x70, + 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, + 0x65, 0x12, 0x30, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, + 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x03, + 0x65, 0x72, 0x72, 0x22, 0x43, 0x0a, 0x0d, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x6c, 0x6f, 0x62, + 0x4c, 0x69, 0x73, 0x74, 0x12, 0x32, 0x0a, 0x04, 0x62, 0x6c, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, + 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x6c, + 0x6f, 0x62, 0x52, 0x04, 0x62, 0x6c, 0x6f, 0x62, 0x22, 0xa7, 0x09, 0x0a, 0x08, 0x5a, 0x49, 0x6e, + 0x66, 0x6f, 0x4d, 0x73, 0x67, 0x12, 0x35, 0x0a, 0x05, 0x7a, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, + 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x05, 0x7a, 0x74, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x64, 0x65, 0x76, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64, 0x65, 0x76, + 0x49, 0x64, 0x12, 0x38, 0x0a, 0x05, 0x64, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, + 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, + 0x69, 0x63, 0x65, 0x48, 0x00, 0x52, 0x05, 0x64, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x35, 0x0a, 0x05, + 0x61, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x41, 0x70, 0x70, 0x48, 0x00, 0x52, 0x05, 0x61, 0x69, + 0x6e, 0x66, 0x6f, 0x12, 0x43, 0x0a, 0x06, 0x6e, 0x69, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x0c, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x48, 0x00, + 0x52, 0x06, 0x6e, 0x69, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x38, 0x0a, 0x05, 0x76, 0x69, 0x6e, 0x66, + 0x6f, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, + 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, + 0x6e, 0x66, 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x48, 0x00, 0x52, 0x05, 0x76, 0x69, 0x6e, + 0x66, 0x6f, 0x12, 0x3d, 0x0a, 0x05, 0x63, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x0e, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, + 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x54, 0x72, 0x65, 0x65, 0x48, 0x00, 0x52, 0x05, 0x63, 0x69, 0x6e, 0x66, + 0x6f, 0x12, 0x3a, 0x0a, 0x05, 0x62, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x6c, 0x6f, 0x62, - 0x52, 0x04, 0x62, 0x6c, 0x6f, 0x62, 0x22, 0xa7, 0x09, 0x0a, 0x08, 0x5a, 0x49, 0x6e, 0x66, 0x6f, - 0x4d, 0x73, 0x67, 0x12, 0x35, 0x0a, 0x05, 0x7a, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, - 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, - 0x70, 0x65, 0x73, 0x52, 0x05, 0x7a, 0x74, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, - 0x76, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x64, 0x65, 0x76, 0x49, 0x64, - 0x12, 0x38, 0x0a, 0x05, 0x64, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, - 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x44, 0x65, 0x76, 0x69, 0x63, - 0x65, 0x48, 0x00, 0x52, 0x05, 0x64, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x35, 0x0a, 0x05, 0x61, 0x69, - 0x6e, 0x66, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, - 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x41, 0x70, 0x70, 0x48, 0x00, 0x52, 0x05, 0x61, 0x69, 0x6e, 0x66, - 0x6f, 0x12, 0x43, 0x0a, 0x06, 0x6e, 0x69, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x0c, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, - 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x48, 0x00, 0x52, 0x06, - 0x6e, 0x69, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x38, 0x0a, 0x05, 0x76, 0x69, 0x6e, 0x66, 0x6f, 0x18, - 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, - 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, - 0x6f, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x48, 0x00, 0x52, 0x05, 0x76, 0x69, 0x6e, 0x66, 0x6f, - 0x12, 0x3d, 0x0a, 0x05, 0x63, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, - 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x54, 0x72, 0x65, 0x65, 0x48, 0x00, 0x52, 0x05, 0x63, 0x69, 0x6e, 0x66, 0x6f, 0x12, - 0x3a, 0x0a, 0x05, 0x62, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, + 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x05, 0x62, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x45, 0x0a, + 0x07, 0x61, 0x6d, 0x64, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, - 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x4c, 0x69, - 0x73, 0x74, 0x48, 0x00, 0x52, 0x05, 0x62, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x45, 0x0a, 0x07, 0x61, - 0x6d, 0x64, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, - 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, - 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x41, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, 0x4d, - 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x07, 0x61, 0x6d, 0x64, 0x69, 0x6e, - 0x66, 0x6f, 0x12, 0x3c, 0x0a, 0x06, 0x65, 0x76, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x11, 0x20, 0x01, + 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x41, 0x70, 0x70, 0x49, 0x6e, 0x73, + 0x74, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x48, 0x00, 0x52, 0x07, 0x61, 0x6d, 0x64, + 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x3c, 0x0a, 0x06, 0x65, 0x76, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x11, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, + 0x45, 0x64, 0x67, 0x65, 0x76, 0x69, 0x65, 0x77, 0x48, 0x00, 0x52, 0x06, 0x65, 0x76, 0x69, 0x6e, + 0x66, 0x6f, 0x12, 0x3c, 0x0a, 0x06, 0x68, 0x77, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, - 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x45, 0x64, - 0x67, 0x65, 0x76, 0x69, 0x65, 0x77, 0x48, 0x00, 0x52, 0x06, 0x65, 0x76, 0x69, 0x6e, 0x66, 0x6f, - 0x12, 0x3c, 0x0a, 0x06, 0x68, 0x77, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, - 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x61, 0x72, 0x64, - 0x77, 0x61, 0x72, 0x65, 0x48, 0x00, 0x52, 0x06, 0x68, 0x77, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x3e, - 0x0a, 0x07, 0x6c, 0x6f, 0x63, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, - 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x6f, 0x63, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x47, - 0x0a, 0x09, 0x70, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x14, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, - 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x50, 0x61, 0x74, - 0x63, 0x68, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x48, 0x00, 0x52, 0x09, 0x70, 0x61, - 0x74, 0x63, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4a, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, - 0x6f, 0x64, 0x65, 0x12, 0x47, 0x0a, 0x0b, 0x6e, 0x74, 0x70, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, - 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, - 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x54, 0x50, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x48, 0x00, - 0x52, 0x0a, 0x6e, 0x74, 0x70, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x4a, 0x0a, 0x0c, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x18, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, - 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4b, 0x75, - 0x62, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x48, 0x00, 0x52, 0x0b, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x63, 0x0a, 0x13, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, - 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, - 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, - 0x6f, 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x11, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x3c, 0x0a, - 0x0b, 0x61, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0b, - 0x61, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x0d, 0x0a, 0x0b, 0x49, - 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4a, 0x04, 0x08, 0x15, 0x10, 0x16, - 0x22, 0x76, 0x0a, 0x0c, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, - 0x12, 0x3a, 0x0a, 0x18, 0x48, 0x57, 0x41, 0x73, 0x73, 0x69, 0x73, 0x74, 0x65, 0x64, 0x56, 0x69, - 0x72, 0x74, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x18, 0x48, 0x57, 0x41, 0x73, 0x73, 0x69, 0x73, 0x74, 0x65, 0x64, 0x56, 0x69, - 0x72, 0x74, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x10, - 0x49, 0x4f, 0x56, 0x69, 0x72, 0x74, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x49, 0x4f, 0x56, 0x69, 0x72, 0x74, 0x75, 0x61, - 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x7c, 0x0a, 0x14, 0x5a, 0x49, 0x6e, 0x66, - 0x6f, 0x41, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, - 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x75, 0x75, 0x69, 0x64, 0x12, 0x3c, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x28, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, - 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, - 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0xe2, 0x01, 0x0a, 0x0d, 0x5a, 0x49, 0x6e, 0x66, 0x6f, - 0x45, 0x64, 0x67, 0x65, 0x76, 0x69, 0x65, 0x77, 0x12, 0x3b, 0x0a, 0x0b, 0x65, 0x78, 0x70, 0x69, - 0x72, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x61, + 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x48, 0x00, 0x52, 0x06, 0x68, 0x77, 0x69, 0x6e, 0x66, 0x6f, + 0x12, 0x3e, 0x0a, 0x07, 0x6c, 0x6f, 0x63, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x13, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x22, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, + 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x6f, 0x63, 0x69, 0x6e, 0x66, 0x6f, + 0x12, 0x47, 0x0a, 0x09, 0x70, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x14, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, + 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x50, + 0x61, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x48, 0x00, 0x52, 0x09, + 0x70, 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4a, 0x0a, 0x0c, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, + 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x47, 0x0a, 0x0b, 0x6e, 0x74, 0x70, 0x5f, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x73, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, + 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, + 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4e, 0x54, 0x50, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, + 0x48, 0x00, 0x52, 0x0a, 0x6e, 0x74, 0x70, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x4a, + 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x18, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, 0x6e, 0x66, 0x6f, + 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x48, 0x00, 0x52, 0x0b, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x63, 0x0a, 0x13, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x69, 0x6e, 0x66, + 0x6f, 0x18, 0x19, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, + 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x5a, 0x49, + 0x6e, 0x66, 0x6f, 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x11, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x3c, 0x0a, 0x0b, 0x61, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x0b, 0x61, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x0d, 0x0a, + 0x0b, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4a, 0x04, 0x08, 0x15, + 0x10, 0x16, 0x22, 0x76, 0x0a, 0x0c, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, + 0x65, 0x73, 0x12, 0x3a, 0x0a, 0x18, 0x48, 0x57, 0x41, 0x73, 0x73, 0x69, 0x73, 0x74, 0x65, 0x64, + 0x56, 0x69, 0x72, 0x74, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, 0x48, 0x57, 0x41, 0x73, 0x73, 0x69, 0x73, 0x74, 0x65, 0x64, + 0x56, 0x69, 0x72, 0x74, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2a, + 0x0a, 0x10, 0x49, 0x4f, 0x56, 0x69, 0x72, 0x74, 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x49, 0x4f, 0x56, 0x69, 0x72, 0x74, + 0x75, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x7c, 0x0a, 0x14, 0x5a, 0x49, + 0x6e, 0x66, 0x6f, 0x41, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, + 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x3c, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x28, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x70, 0x70, 0x49, 0x6e, + 0x73, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0xe2, 0x01, 0x0a, 0x0d, 0x5a, 0x49, 0x6e, + 0x66, 0x6f, 0x45, 0x64, 0x67, 0x65, 0x76, 0x69, 0x65, 0x77, 0x12, 0x3b, 0x0a, 0x0b, 0x65, 0x78, + 0x70, 0x69, 0x72, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x65, 0x78, 0x70, + 0x69, 0x72, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x0c, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, - 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x0c, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, - 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x64, 0x65, - 0x76, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x44, 0x65, - 0x76, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x61, 0x70, 0x70, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x41, 0x70, 0x70, 0x12, 0x1b, - 0x0a, 0x09, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x78, 0x74, 0x22, 0xe8, 0x03, 0x0a, 0x0d, - 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, - 0x08, 0x6c, 0x61, 0x74, 0x69, 0x74, 0x75, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, - 0x08, 0x6c, 0x61, 0x74, 0x69, 0x74, 0x75, 0x64, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6c, 0x6f, 0x6e, - 0x67, 0x69, 0x74, 0x75, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x6c, 0x6f, - 0x6e, 0x67, 0x69, 0x74, 0x75, 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x6c, 0x74, 0x69, 0x74, - 0x75, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x61, 0x6c, 0x74, 0x69, 0x74, - 0x75, 0x64, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x75, 0x74, 0x63, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x75, 0x74, 0x63, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x12, 0x5a, 0x0a, 0x16, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x6f, 0x6e, 0x74, - 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, - 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4c, 0x6f, 0x63, 0x52, 0x65, - 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x15, 0x68, 0x6f, 0x72, 0x69, 0x7a, - 0x6f, 0x6e, 0x74, 0x61, 0x6c, 0x52, 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, - 0x12, 0x56, 0x0a, 0x14, 0x76, 0x65, 0x72, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x6c, - 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, - 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, - 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4c, 0x6f, 0x63, 0x52, 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, - 0x69, 0x74, 0x79, 0x52, 0x13, 0x76, 0x65, 0x72, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x52, 0x65, 0x6c, - 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x35, 0x0a, 0x16, 0x68, 0x6f, 0x72, 0x69, - 0x7a, 0x6f, 0x6e, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, - 0x74, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x02, 0x52, 0x15, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x6f, - 0x6e, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x12, - 0x31, 0x0a, 0x14, 0x76, 0x65, 0x72, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x63, 0x65, - 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x02, 0x52, 0x13, 0x76, - 0x65, 0x72, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x55, 0x6e, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, - 0x74, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, - 0x6c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x22, 0x96, 0x02, 0x0a, 0x1c, 0x5a, 0x49, 0x6e, 0x66, 0x6f, - 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x75, 0x72, 0x72, 0x65, - 0x6e, 0x74, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, - 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x3b, 0x0a, 0x09, 0x63, 0x6f, - 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, - 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, - 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x6f, - 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x41, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, - 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, - 0x62, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x34, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, + 0x64, 0x65, 0x76, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x44, 0x65, 0x76, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x61, 0x70, 0x70, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x41, 0x70, 0x70, + 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x74, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x45, 0x78, 0x74, 0x22, 0xe8, 0x03, + 0x0a, 0x0d, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x1a, 0x0a, 0x08, 0x6c, 0x61, 0x74, 0x69, 0x74, 0x75, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x08, 0x6c, 0x61, 0x74, 0x69, 0x74, 0x75, 0x64, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6c, + 0x6f, 0x6e, 0x67, 0x69, 0x74, 0x75, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, + 0x6c, 0x6f, 0x6e, 0x67, 0x69, 0x74, 0x75, 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x6c, 0x74, + 0x69, 0x74, 0x75, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x61, 0x6c, 0x74, + 0x69, 0x74, 0x75, 0x64, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x75, 0x74, 0x63, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x75, 0x74, 0x63, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x5a, 0x0a, 0x16, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x6f, + 0x6e, 0x74, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4c, 0x6f, 0x63, + 0x52, 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x15, 0x68, 0x6f, 0x72, + 0x69, 0x7a, 0x6f, 0x6e, 0x74, 0x61, 0x6c, 0x52, 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, + 0x74, 0x79, 0x12, 0x56, 0x0a, 0x14, 0x76, 0x65, 0x72, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x72, + 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x23, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, + 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4c, 0x6f, 0x63, 0x52, 0x65, 0x6c, 0x69, 0x61, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x13, 0x76, 0x65, 0x72, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x52, + 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x35, 0x0a, 0x16, 0x68, 0x6f, + 0x72, 0x69, 0x7a, 0x6f, 0x6e, 0x74, 0x61, 0x6c, 0x5f, 0x75, 0x6e, 0x63, 0x65, 0x72, 0x74, 0x61, + 0x69, 0x6e, 0x74, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x02, 0x52, 0x15, 0x68, 0x6f, 0x72, 0x69, + 0x7a, 0x6f, 0x6e, 0x74, 0x61, 0x6c, 0x55, 0x6e, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, + 0x79, 0x12, 0x31, 0x0a, 0x14, 0x76, 0x65, 0x72, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x75, 0x6e, + 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x02, 0x52, + 0x13, 0x76, 0x65, 0x72, 0x74, 0x69, 0x63, 0x61, 0x6c, 0x55, 0x6e, 0x63, 0x65, 0x72, 0x74, 0x61, + 0x69, 0x6e, 0x74, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6c, 0x6f, 0x67, 0x69, + 0x63, 0x61, 0x6c, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x22, 0x96, 0x02, 0x0a, 0x1c, 0x5a, 0x49, 0x6e, + 0x66, 0x6f, 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x3b, 0x0a, 0x09, + 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x1d, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, + 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x52, 0x09, + 0x63, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x12, 0x41, 0x0a, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x22, - 0x82, 0x03, 0x0a, 0x10, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, - 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x4e, 0x6f, - 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x51, 0x0a, - 0x0f, 0x70, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, + 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x34, 0x0a, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x72, + 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, + 0x6f, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, + 0x64, 0x22, 0x82, 0x03, 0x0a, 0x10, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x4b, 0x75, 0x62, 0x65, 0x43, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, + 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, 0x62, 0x65, + 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, + 0x51, 0x0a, 0x0f, 0x70, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, + 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, + 0x75, 0x62, 0x65, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x70, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x70, 0x61, 0x63, + 0x65, 0x73, 0x12, 0x41, 0x0a, 0x08, 0x65, 0x76, 0x65, 0x5f, 0x61, 0x70, 0x70, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, + 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x45, + 0x56, 0x45, 0x41, 0x70, 0x70, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x65, 0x76, + 0x65, 0x41, 0x70, 0x70, 0x73, 0x12, 0x3e, 0x0a, 0x07, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, 0x62, - 0x65, 0x50, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x0d, 0x70, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x73, - 0x12, 0x41, 0x0a, 0x08, 0x65, 0x76, 0x65, 0x5f, 0x61, 0x70, 0x70, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, - 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x45, 0x56, 0x45, - 0x41, 0x70, 0x70, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x65, 0x76, 0x65, 0x41, - 0x70, 0x70, 0x73, 0x12, 0x3e, 0x0a, 0x07, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, - 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, 0x62, 0x65, 0x53, - 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x49, 0x64, 0x12, 0x40, 0x0a, 0x0b, 0x65, 0x76, 0x65, 0x5f, 0x76, 0x6d, 0x5f, 0x61, 0x70, 0x70, - 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, - 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x4b, 0x75, - 0x62, 0x65, 0x56, 0x4d, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x65, 0x76, 0x65, 0x56, 0x6d, - 0x41, 0x70, 0x70, 0x73, 0x22, 0xea, 0x02, 0x0a, 0x0d, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x48, 0x61, - 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x12, 0x3e, 0x0a, 0x05, 0x64, 0x69, 0x73, 0x6b, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, - 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x02, 0x18, 0x01, 0x52, - 0x05, 0x64, 0x69, 0x73, 0x6b, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x76, 0x65, 0x5f, 0x72, 0x65, - 0x6c, 0x65, 0x61, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x76, 0x65, - 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x09, 0x69, 0x6e, 0x76, 0x65, 0x6e, - 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, 0x72, 0x67, - 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, - 0x2e, 0x48, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x76, 0x65, 0x6e, 0x74, 0x6f, - 0x72, 0x79, 0x52, 0x09, 0x69, 0x6e, 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x23, 0x0a, - 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x5f, 0x66, 0x6c, 0x61, 0x76, 0x6f, 0x72, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x46, 0x6c, 0x61, 0x76, - 0x6f, 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x5f, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6b, 0x65, 0x72, 0x6e, - 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x76, 0x65, - 0x5f, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x65, 0x76, 0x65, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x1c, 0x0a, 0x09, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x6b, 0x65, - 0x72, 0x6e, 0x65, 0x6c, 0x5f, 0x63, 0x6d, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x43, 0x6d, 0x64, 0x6c, 0x69, 0x6e, - 0x65, 0x2a, 0x75, 0x0a, 0x11, 0x44, 0x65, 0x70, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x74, - 0x65, 0x6d, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x65, 0x70, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, 0x6d, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x10, 0x00, 0x12, 0x16, - 0x0a, 0x12, 0x44, 0x65, 0x70, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, 0x6d, 0x47, - 0x61, 0x75, 0x67, 0x65, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x44, 0x65, 0x70, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, 0x6d, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x10, 0x02, + 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x73, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x49, 0x64, 0x12, 0x40, 0x0a, 0x0b, 0x65, 0x76, 0x65, 0x5f, 0x76, 0x6d, 0x5f, 0x61, + 0x70, 0x70, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x72, 0x67, 0x2e, + 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, + 0x4b, 0x75, 0x62, 0x65, 0x56, 0x4d, 0x49, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x65, 0x76, 0x65, + 0x56, 0x6d, 0x41, 0x70, 0x70, 0x73, 0x22, 0xea, 0x02, 0x0a, 0x0d, 0x5a, 0x49, 0x6e, 0x66, 0x6f, + 0x48, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x12, 0x3e, 0x0a, 0x05, 0x64, 0x69, 0x73, 0x6b, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, + 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x74, + 0x6f, 0x72, 0x61, 0x67, 0x65, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x42, 0x02, 0x18, + 0x01, 0x52, 0x05, 0x64, 0x69, 0x73, 0x6b, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x76, 0x65, 0x5f, + 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, + 0x76, 0x65, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x09, 0x69, 0x6e, 0x76, + 0x65, 0x6e, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6f, + 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x48, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x49, 0x6e, 0x76, 0x65, 0x6e, + 0x74, 0x6f, 0x72, 0x79, 0x52, 0x09, 0x69, 0x6e, 0x76, 0x65, 0x6e, 0x74, 0x6f, 0x72, 0x79, 0x12, + 0x23, 0x0a, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x5f, 0x66, 0x6c, 0x61, 0x76, 0x6f, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x46, 0x6c, + 0x61, 0x76, 0x6f, 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6b, 0x65, + 0x72, 0x6e, 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x65, + 0x76, 0x65, 0x5f, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x65, 0x76, 0x65, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x1c, + 0x0a, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, + 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x5f, 0x63, 0x6d, 0x64, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x43, 0x6d, 0x64, 0x6c, + 0x69, 0x6e, 0x65, 0x2a, 0x75, 0x0a, 0x11, 0x44, 0x65, 0x70, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, + 0x49, 0x74, 0x65, 0x6d, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x65, 0x70, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, 0x6d, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x65, 0x70, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, - 0x6d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x10, 0x03, 0x2a, 0xb0, 0x02, 0x0a, 0x0a, 0x5a, 0x49, 0x6e, - 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x09, 0x0a, 0x05, 0x5a, 0x69, 0x4e, 0x6f, 0x70, - 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x5a, 0x69, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x10, 0x01, - 0x12, 0x09, 0x0a, 0x05, 0x5a, 0x69, 0x41, 0x70, 0x70, 0x10, 0x03, 0x12, 0x15, 0x0a, 0x11, 0x5a, - 0x69, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x10, 0x06, 0x12, 0x0c, 0x0a, 0x08, 0x5a, 0x69, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x10, 0x07, - 0x12, 0x11, 0x0a, 0x0d, 0x5a, 0x69, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x72, 0x65, - 0x65, 0x10, 0x08, 0x12, 0x0e, 0x0a, 0x0a, 0x5a, 0x69, 0x42, 0x6c, 0x6f, 0x62, 0x4c, 0x69, 0x73, - 0x74, 0x10, 0x09, 0x12, 0x15, 0x0a, 0x11, 0x5a, 0x69, 0x41, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, - 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x10, 0x0a, 0x12, 0x0e, 0x0a, 0x0a, 0x5a, 0x69, - 0x48, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x10, 0x0b, 0x12, 0x0e, 0x0a, 0x0a, 0x5a, 0x69, - 0x45, 0x64, 0x67, 0x65, 0x76, 0x69, 0x65, 0x77, 0x10, 0x0c, 0x12, 0x0e, 0x0a, 0x0a, 0x5a, 0x69, - 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x0d, 0x12, 0x13, 0x0a, 0x0f, 0x5a, 0x69, - 0x50, 0x61, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x10, 0x0e, 0x12, - 0x10, 0x0a, 0x0c, 0x5a, 0x69, 0x4e, 0x54, 0x50, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x10, - 0x10, 0x12, 0x11, 0x0a, 0x0d, 0x5a, 0x69, 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x10, 0x11, 0x12, 0x1d, 0x0a, 0x19, 0x5a, 0x69, 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x10, 0x12, 0x22, 0x04, 0x08, 0x02, 0x10, 0x02, 0x22, 0x04, 0x08, 0x04, 0x10, 0x04, 0x22, - 0x04, 0x08, 0x05, 0x10, 0x05, 0x22, 0x04, 0x08, 0x0f, 0x10, 0x0f, 0x2a, 0x91, 0x03, 0x0a, 0x08, - 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, - 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x49, 0x54, 0x49, 0x41, 0x4c, - 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x53, - 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x4f, 0x57, 0x4e, - 0x4c, 0x4f, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x45, 0x4c, 0x49, - 0x56, 0x45, 0x52, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x4e, 0x53, 0x54, 0x41, - 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x4f, 0x4f, 0x54, 0x49, 0x4e, - 0x47, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x07, - 0x12, 0x0b, 0x0a, 0x07, 0x48, 0x41, 0x4c, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x08, 0x12, 0x0a, 0x0a, - 0x06, 0x48, 0x41, 0x4c, 0x54, 0x45, 0x44, 0x10, 0x09, 0x12, 0x0e, 0x0a, 0x0a, 0x52, 0x45, 0x53, - 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x0a, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x55, 0x52, - 0x47, 0x49, 0x4e, 0x47, 0x10, 0x0b, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x53, 0x4f, 0x4c, 0x56, - 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x41, 0x47, 0x10, 0x0c, 0x12, 0x10, 0x0a, 0x0c, 0x52, 0x45, 0x53, - 0x4f, 0x4c, 0x56, 0x45, 0x44, 0x5f, 0x54, 0x41, 0x47, 0x10, 0x0d, 0x12, 0x13, 0x0a, 0x0f, 0x43, - 0x52, 0x45, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x56, 0x4f, 0x4c, 0x55, 0x4d, 0x45, 0x10, 0x0e, - 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x4f, 0x4c, 0x55, - 0x4d, 0x45, 0x10, 0x0f, 0x12, 0x0d, 0x0a, 0x09, 0x56, 0x45, 0x52, 0x49, 0x46, 0x59, 0x49, 0x4e, - 0x47, 0x10, 0x10, 0x12, 0x0c, 0x0a, 0x08, 0x56, 0x45, 0x52, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, - 0x11, 0x12, 0x0b, 0x0a, 0x07, 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x12, 0x12, 0x0a, - 0x0a, 0x06, 0x4c, 0x4f, 0x41, 0x44, 0x45, 0x44, 0x10, 0x13, 0x12, 0x18, 0x0a, 0x14, 0x41, 0x57, - 0x41, 0x49, 0x54, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4e, - 0x43, 0x45, 0x10, 0x14, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x15, 0x12, - 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x41, 0x52, 0x54, 0x5f, 0x44, 0x45, 0x4c, 0x41, 0x59, 0x45, 0x44, - 0x10, 0x16, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x17, 0x12, - 0x0e, 0x0a, 0x0a, 0x53, 0x43, 0x48, 0x45, 0x44, 0x55, 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x18, 0x2a, - 0x4e, 0x0a, 0x16, 0x48, 0x77, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4d, 0x6f, 0x64, - 0x75, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, - 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x54, 0x46, 0x4f, 0x55, - 0x4e, 0x44, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x44, - 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x45, 0x4e, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, 0x03, 0x2a, - 0x88, 0x01, 0x0a, 0x13, 0x44, 0x61, 0x74, 0x61, 0x53, 0x65, 0x63, 0x41, 0x74, 0x52, 0x65, 0x73, - 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x44, 0x41, 0x54, 0x41, 0x53, - 0x45, 0x43, 0x5f, 0x41, 0x54, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, - 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x44, 0x41, 0x54, 0x41, 0x53, 0x45, 0x43, 0x5f, - 0x41, 0x54, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x5f, 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x44, - 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x44, 0x41, 0x54, 0x41, 0x53, 0x45, 0x43, 0x5f, 0x41, 0x54, - 0x5f, 0x52, 0x45, 0x53, 0x54, 0x5f, 0x45, 0x4e, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, 0x02, 0x12, - 0x19, 0x0a, 0x15, 0x44, 0x41, 0x54, 0x41, 0x53, 0x45, 0x43, 0x5f, 0x41, 0x54, 0x5f, 0x52, 0x45, - 0x53, 0x54, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x2a, 0x3f, 0x0a, 0x09, 0x50, 0x43, - 0x52, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x43, 0x52, 0x5f, 0x55, - 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x43, 0x52, 0x5f, - 0x45, 0x4e, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x50, 0x43, 0x52, - 0x5f, 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x4d, 0x0a, 0x07, 0x53, - 0x69, 0x6d, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x49, 0x4d, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x15, 0x0a, 0x11, 0x53, 0x49, 0x4d, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x48, 0x59, - 0x53, 0x49, 0x43, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x49, 0x4d, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x45, 0x53, 0x49, 0x4d, 0x10, 0x02, 0x2a, 0xa0, 0x02, 0x0a, 0x17, 0x5a, - 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6e, - 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2a, 0x0a, 0x26, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, + 0x6d, 0x47, 0x61, 0x75, 0x67, 0x65, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x44, 0x65, 0x70, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, 0x74, 0x65, 0x6d, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, + 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x44, 0x65, 0x70, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x49, + 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x10, 0x03, 0x2a, 0xb0, 0x02, 0x0a, 0x0a, 0x5a, + 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x09, 0x0a, 0x05, 0x5a, 0x69, 0x4e, + 0x6f, 0x70, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x5a, 0x69, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, + 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x5a, 0x69, 0x41, 0x70, 0x70, 0x10, 0x03, 0x12, 0x15, 0x0a, + 0x11, 0x5a, 0x69, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, + 0x63, 0x65, 0x10, 0x06, 0x12, 0x0c, 0x0a, 0x08, 0x5a, 0x69, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x10, 0x07, 0x12, 0x11, 0x0a, 0x0d, 0x5a, 0x69, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, + 0x72, 0x65, 0x65, 0x10, 0x08, 0x12, 0x0e, 0x0a, 0x0a, 0x5a, 0x69, 0x42, 0x6c, 0x6f, 0x62, 0x4c, + 0x69, 0x73, 0x74, 0x10, 0x09, 0x12, 0x15, 0x0a, 0x11, 0x5a, 0x69, 0x41, 0x70, 0x70, 0x49, 0x6e, + 0x73, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x10, 0x0a, 0x12, 0x0e, 0x0a, 0x0a, + 0x5a, 0x69, 0x48, 0x61, 0x72, 0x64, 0x77, 0x61, 0x72, 0x65, 0x10, 0x0b, 0x12, 0x0e, 0x0a, 0x0a, + 0x5a, 0x69, 0x45, 0x64, 0x67, 0x65, 0x76, 0x69, 0x65, 0x77, 0x10, 0x0c, 0x12, 0x0e, 0x0a, 0x0a, + 0x5a, 0x69, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x0d, 0x12, 0x13, 0x0a, 0x0f, + 0x5a, 0x69, 0x50, 0x61, 0x74, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x10, + 0x0e, 0x12, 0x10, 0x0a, 0x0c, 0x5a, 0x69, 0x4e, 0x54, 0x50, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x73, 0x10, 0x10, 0x12, 0x11, 0x0a, 0x0d, 0x5a, 0x69, 0x4b, 0x75, 0x62, 0x65, 0x43, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x10, 0x11, 0x12, 0x1d, 0x0a, 0x19, 0x5a, 0x69, 0x4b, 0x75, 0x62, 0x65, + 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x10, 0x12, 0x22, 0x04, 0x08, 0x02, 0x10, 0x02, 0x22, 0x04, 0x08, 0x04, 0x10, + 0x04, 0x22, 0x04, 0x08, 0x05, 0x10, 0x05, 0x22, 0x04, 0x08, 0x0f, 0x10, 0x0f, 0x2a, 0x91, 0x03, + 0x0a, 0x08, 0x5a, 0x53, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, + 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x49, 0x54, 0x49, + 0x41, 0x4c, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, + 0x5f, 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x4f, + 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x45, 0x44, 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x45, + 0x4c, 0x49, 0x56, 0x45, 0x52, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x4e, 0x53, + 0x54, 0x41, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x4f, 0x4f, 0x54, + 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, + 0x10, 0x07, 0x12, 0x0b, 0x0a, 0x07, 0x48, 0x41, 0x4c, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x08, 0x12, + 0x0a, 0x0a, 0x06, 0x48, 0x41, 0x4c, 0x54, 0x45, 0x44, 0x10, 0x09, 0x12, 0x0e, 0x0a, 0x0a, 0x52, + 0x45, 0x53, 0x54, 0x41, 0x52, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x0a, 0x12, 0x0b, 0x0a, 0x07, 0x50, + 0x55, 0x52, 0x47, 0x49, 0x4e, 0x47, 0x10, 0x0b, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x53, 0x4f, + 0x4c, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x41, 0x47, 0x10, 0x0c, 0x12, 0x10, 0x0a, 0x0c, 0x52, + 0x45, 0x53, 0x4f, 0x4c, 0x56, 0x45, 0x44, 0x5f, 0x54, 0x41, 0x47, 0x10, 0x0d, 0x12, 0x13, 0x0a, + 0x0f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x56, 0x4f, 0x4c, 0x55, 0x4d, 0x45, + 0x10, 0x0e, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x4f, + 0x4c, 0x55, 0x4d, 0x45, 0x10, 0x0f, 0x12, 0x0d, 0x0a, 0x09, 0x56, 0x45, 0x52, 0x49, 0x46, 0x59, + 0x49, 0x4e, 0x47, 0x10, 0x10, 0x12, 0x0c, 0x0a, 0x08, 0x56, 0x45, 0x52, 0x49, 0x46, 0x49, 0x45, + 0x44, 0x10, 0x11, 0x12, 0x0b, 0x0a, 0x07, 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x12, + 0x12, 0x0a, 0x0a, 0x06, 0x4c, 0x4f, 0x41, 0x44, 0x45, 0x44, 0x10, 0x13, 0x12, 0x18, 0x0a, 0x14, + 0x41, 0x57, 0x41, 0x49, 0x54, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x49, 0x4e, 0x53, 0x54, + 0x41, 0x4e, 0x43, 0x45, 0x10, 0x14, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, + 0x15, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x54, 0x41, 0x52, 0x54, 0x5f, 0x44, 0x45, 0x4c, 0x41, 0x59, + 0x45, 0x44, 0x10, 0x16, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, + 0x17, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x43, 0x48, 0x45, 0x44, 0x55, 0x4c, 0x49, 0x4e, 0x47, 0x10, + 0x18, 0x2a, 0x4e, 0x0a, 0x16, 0x48, 0x77, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4d, + 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x55, + 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x4f, 0x54, 0x46, + 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, + 0x45, 0x44, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x45, 0x4e, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, + 0x03, 0x2a, 0x88, 0x01, 0x0a, 0x13, 0x44, 0x61, 0x74, 0x61, 0x53, 0x65, 0x63, 0x41, 0x74, 0x52, + 0x65, 0x73, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x17, 0x44, 0x41, 0x54, + 0x41, 0x53, 0x45, 0x43, 0x5f, 0x41, 0x54, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x5f, 0x55, 0x4e, 0x4b, + 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x44, 0x41, 0x54, 0x41, 0x53, 0x45, + 0x43, 0x5f, 0x41, 0x54, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x5f, 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, + 0x45, 0x44, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x44, 0x41, 0x54, 0x41, 0x53, 0x45, 0x43, 0x5f, + 0x41, 0x54, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x5f, 0x45, 0x4e, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, + 0x02, 0x12, 0x19, 0x0a, 0x15, 0x44, 0x41, 0x54, 0x41, 0x53, 0x45, 0x43, 0x5f, 0x41, 0x54, 0x5f, + 0x52, 0x45, 0x53, 0x54, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x2a, 0x3f, 0x0a, 0x09, + 0x50, 0x43, 0x52, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x43, 0x52, + 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x43, + 0x52, 0x5f, 0x45, 0x4e, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x50, + 0x43, 0x52, 0x5f, 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x4d, 0x0a, + 0x07, 0x53, 0x69, 0x6d, 0x54, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x49, 0x4d, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x49, 0x4d, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, + 0x48, 0x59, 0x53, 0x49, 0x43, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x49, 0x4d, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x53, 0x49, 0x4d, 0x10, 0x02, 0x2a, 0xa0, 0x02, 0x0a, + 0x17, 0x5a, 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2a, 0x0a, 0x26, 0x5a, 0x5f, 0x43, 0x45, + 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4e, 0x47, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x26, 0x0a, 0x22, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, + 0x41, 0x52, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x4f, 0x46, 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x28, 0x0a, 0x24, + 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, + 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x41, 0x44, 0x49, 0x4f, + 0x5f, 0x4f, 0x46, 0x46, 0x10, 0x02, 0x12, 0x25, 0x0a, 0x21, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x26, 0x0a, 0x22, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x03, 0x12, 0x33, 0x0a, + 0x2f, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x4f, 0x50, 0x45, 0x52, + 0x41, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, + 0x4e, 0x45, 0x5f, 0x41, 0x4e, 0x44, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x45, 0x44, + 0x10, 0x04, 0x12, 0x2b, 0x0a, 0x27, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, - 0x5f, 0x4f, 0x46, 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x28, 0x0a, 0x24, 0x5a, 0x5f, - 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, - 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x41, 0x44, 0x49, 0x4f, 0x5f, 0x4f, - 0x46, 0x46, 0x10, 0x02, 0x12, 0x25, 0x0a, 0x21, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, - 0x41, 0x52, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x03, 0x12, 0x33, 0x0a, 0x2f, 0x5a, - 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, - 0x5f, 0x41, 0x4e, 0x44, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x45, 0x44, 0x10, 0x04, - 0x12, 0x2b, 0x0a, 0x27, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x4f, - 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, - 0x4e, 0x52, 0x45, 0x43, 0x4f, 0x47, 0x4e, 0x49, 0x5a, 0x45, 0x44, 0x10, 0x05, 0x2a, 0x92, 0x01, - 0x0a, 0x18, 0x5a, 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x72, - 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x2b, 0x0a, 0x27, 0x5a, 0x5f, - 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, - 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x23, 0x0a, 0x1f, 0x5a, 0x5f, 0x43, 0x45, 0x4c, - 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x50, 0x52, - 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x51, 0x4d, 0x49, 0x10, 0x01, 0x12, 0x24, 0x0a, 0x20, + 0x5f, 0x55, 0x4e, 0x52, 0x45, 0x43, 0x4f, 0x47, 0x4e, 0x49, 0x5a, 0x45, 0x44, 0x10, 0x05, 0x2a, + 0x92, 0x01, 0x0a, 0x18, 0x5a, 0x43, 0x65, 0x6c, 0x6c, 0x75, 0x6c, 0x61, 0x72, 0x43, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x2b, 0x0a, 0x27, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, - 0x4f, 0x4c, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x4d, 0x42, 0x49, 0x4d, - 0x10, 0x02, 0x2a, 0xb1, 0x02, 0x0a, 0x0c, 0x5a, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, - 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x45, - 0x42, 0x4f, 0x4f, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x22, 0x0a, 0x1e, 0x5a, 0x44, 0x45, - 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x4d, 0x41, 0x49, 0x4e, 0x54, - 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x10, 0x03, 0x12, 0x21, 0x0a, - 0x1d, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x42, - 0x41, 0x53, 0x45, 0x4f, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x04, - 0x12, 0x19, 0x0a, 0x15, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x45, 0x5f, 0x42, 0x4f, 0x4f, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x24, 0x0a, 0x20, 0x5a, - 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, - 0x50, 0x41, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x4f, 0x57, 0x45, 0x52, 0x4f, 0x46, 0x46, 0x10, - 0x06, 0x12, 0x1e, 0x0a, 0x1a, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x45, 0x5f, 0x50, 0x4f, 0x57, 0x45, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x4f, 0x46, 0x46, 0x10, - 0x07, 0x12, 0x23, 0x0a, 0x1f, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45, 0x44, 0x5f, 0x50, 0x4f, 0x57, 0x45, - 0x52, 0x4f, 0x46, 0x46, 0x10, 0x08, 0x2a, 0xf5, 0x01, 0x0a, 0x0d, 0x53, 0x74, 0x6f, 0x72, 0x61, - 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x54, 0x4f, 0x52, - 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, - 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x4f, 0x52, - 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, - 0x45, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, 0x10, 0x02, - 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x45, 0x44, 0x10, 0x03, 0x12, 0x1a, 0x0a, 0x16, - 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x4f, - 0x46, 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x54, 0x4f, 0x52, - 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x56, 0x41, - 0x49, 0x4c, 0x10, 0x05, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x06, - 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x5f, 0x53, 0x55, 0x53, 0x50, 0x45, 0x4e, 0x44, 0x45, 0x44, 0x10, 0x07, 0x2a, 0xe3, - 0x01, 0x0a, 0x0f, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x61, 0x69, 0x64, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x52, 0x41, - 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, - 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, - 0x5f, 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x30, - 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x52, 0x41, - 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x31, 0x10, 0x02, 0x12, - 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x35, 0x10, 0x03, 0x12, 0x1b, 0x0a, 0x17, - 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x36, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x4f, - 0x52, 0x41, 0x47, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, - 0x41, 0x49, 0x44, 0x37, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, - 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x4f, 0x52, 0x41, - 0x49, 0x44, 0x10, 0x06, 0x2a, 0x6b, 0x0a, 0x0f, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x54, - 0x79, 0x70, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x54, 0x4f, 0x52, 0x41, - 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x55, 0x4e, 0x53, - 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x54, - 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, - 0x45, 0x58, 0x54, 0x34, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x5a, 0x46, 0x53, 0x10, - 0x02, 0x2a, 0x87, 0x07, 0x0a, 0x0d, 0x41, 0x50, 0x49, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, - 0x69, 0x74, 0x79, 0x12, 0x1e, 0x0a, 0x1a, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, - 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, 0x1b, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, - 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x52, 0x45, 0x54, 0x52, 0x59, 0x5f, 0x55, 0x50, 0x44, 0x41, - 0x54, 0x45, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, - 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x53, 0x48, 0x55, 0x54, 0x44, 0x4f, 0x57, 0x4e, 0x10, - 0x02, 0x12, 0x29, 0x0a, 0x25, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, - 0x49, 0x54, 0x59, 0x5f, 0x53, 0x54, 0x41, 0x52, 0x54, 0x5f, 0x44, 0x45, 0x4c, 0x41, 0x59, 0x5f, - 0x49, 0x4e, 0x5f, 0x53, 0x45, 0x43, 0x4f, 0x4e, 0x44, 0x53, 0x10, 0x03, 0x12, 0x1b, 0x0a, 0x17, - 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x45, - 0x44, 0x47, 0x45, 0x56, 0x49, 0x45, 0x57, 0x10, 0x04, 0x12, 0x23, 0x0a, 0x1f, 0x41, 0x50, 0x49, - 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x56, 0x4f, 0x4c, 0x55, - 0x4d, 0x45, 0x5f, 0x53, 0x4e, 0x41, 0x50, 0x53, 0x48, 0x4f, 0x54, 0x53, 0x10, 0x05, 0x12, 0x2b, - 0x0a, 0x27, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, - 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4e, 0x43, - 0x45, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x1c, 0x0a, 0x18, 0x41, - 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x42, 0x4f, - 0x4f, 0x54, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x10, 0x07, 0x12, 0x16, 0x0a, 0x12, 0x41, 0x50, 0x49, - 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x4d, 0x54, 0x55, 0x10, - 0x08, 0x12, 0x26, 0x0a, 0x22, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, - 0x49, 0x54, 0x59, 0x5f, 0x41, 0x44, 0x41, 0x50, 0x54, 0x45, 0x52, 0x5f, 0x55, 0x53, 0x45, 0x52, - 0x5f, 0x4c, 0x41, 0x42, 0x45, 0x4c, 0x53, 0x10, 0x09, 0x12, 0x2f, 0x0a, 0x2b, 0x41, 0x50, 0x49, - 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x45, 0x4e, 0x46, 0x4f, - 0x52, 0x43, 0x45, 0x44, 0x5f, 0x4e, 0x45, 0x54, 0x5f, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x46, 0x41, - 0x43, 0x45, 0x5f, 0x4f, 0x52, 0x44, 0x45, 0x52, 0x10, 0x0a, 0x12, 0x1c, 0x0a, 0x18, 0x41, 0x50, - 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x4e, 0x54, 0x50, - 0x53, 0x5f, 0x46, 0x51, 0x44, 0x4e, 0x10, 0x0b, 0x12, 0x26, 0x0a, 0x22, 0x41, 0x50, 0x49, 0x5f, - 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x57, 0x49, 0x4e, 0x5f, 0x4c, - 0x49, 0x43, 0x5f, 0x50, 0x41, 0x53, 0x53, 0x54, 0x48, 0x52, 0x4f, 0x55, 0x47, 0x48, 0x10, 0x0c, - 0x12, 0x2d, 0x0a, 0x29, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, - 0x54, 0x59, 0x5f, 0x56, 0x4f, 0x4c, 0x55, 0x4d, 0x45, 0x5f, 0x53, 0x4e, 0x41, 0x50, 0x53, 0x48, - 0x4f, 0x54, 0x53, 0x5f, 0x49, 0x4d, 0x4d, 0x45, 0x44, 0x49, 0x41, 0x54, 0x45, 0x10, 0x0d, 0x12, - 0x2b, 0x0a, 0x27, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, - 0x59, 0x5f, 0x45, 0x4e, 0x43, 0x52, 0x59, 0x50, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x54, 0x43, - 0x48, 0x5f, 0x45, 0x4e, 0x56, 0x45, 0x4c, 0x4f, 0x50, 0x45, 0x10, 0x0e, 0x12, 0x2a, 0x0a, 0x26, - 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x53, - 0x49, 0x4e, 0x47, 0x4c, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x43, 0x4b, 0x5f, 0x49, 0x50, 0x5f, 0x4e, - 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x10, 0x0f, 0x12, 0x29, 0x0a, 0x25, 0x41, 0x50, 0x49, 0x5f, - 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, - 0x4c, 0x41, 0x52, 0x5f, 0x41, 0x54, 0x54, 0x41, 0x43, 0x48, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x49, - 0x47, 0x10, 0x10, 0x12, 0x2a, 0x0a, 0x26, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, - 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x45, 0x44, 0x47, 0x45, 0x56, 0x49, 0x45, 0x57, 0x5f, 0x41, - 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x11, 0x12, - 0x1f, 0x0a, 0x1b, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, - 0x59, 0x5f, 0x44, 0x49, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x5f, 0x56, 0x54, 0x50, 0x4d, 0x10, 0x12, - 0x12, 0x2a, 0x0a, 0x26, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, - 0x54, 0x59, 0x5f, 0x4c, 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x43, 0x4f, - 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x13, 0x12, 0x1f, 0x0a, 0x1b, - 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x53, - 0x4d, 0x41, 0x52, 0x54, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x52, 0x54, 0x10, 0x14, 0x12, 0x26, 0x0a, - 0x22, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, - 0x52, 0x45, 0x50, 0x4f, 0x52, 0x54, 0x5f, 0x54, 0x50, 0x4d, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, - 0x4c, 0x4f, 0x47, 0x10, 0x15, 0x12, 0x34, 0x0a, 0x30, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, - 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, 0x54, - 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4e, 0x45, 0x54, 0x5f, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x46, 0x41, - 0x43, 0x45, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x16, 0x2a, 0xb9, 0x03, 0x0a, 0x0a, - 0x42, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x17, 0x42, 0x4f, - 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x42, 0x4f, 0x4f, 0x54, 0x5f, - 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x46, 0x49, 0x52, 0x53, 0x54, 0x10, 0x01, 0x12, 0x1a, - 0x0a, 0x16, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x52, 0x45, - 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x43, 0x4d, 0x44, 0x10, 0x02, 0x12, 0x16, 0x0a, 0x12, 0x42, 0x4f, - 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, - 0x10, 0x03, 0x12, 0x18, 0x0a, 0x14, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, - 0x4e, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, - 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x44, 0x49, 0x53, 0x43, - 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x42, 0x4f, 0x4f, 0x54, - 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x06, 0x12, - 0x13, 0x0a, 0x0f, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x4f, - 0x4f, 0x4d, 0x10, 0x07, 0x12, 0x1d, 0x0a, 0x19, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, - 0x53, 0x4f, 0x4e, 0x5f, 0x57, 0x41, 0x54, 0x43, 0x48, 0x44, 0x4f, 0x47, 0x5f, 0x48, 0x55, 0x4e, - 0x47, 0x10, 0x08, 0x12, 0x1c, 0x0a, 0x18, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, - 0x4f, 0x4e, 0x5f, 0x57, 0x41, 0x54, 0x43, 0x48, 0x44, 0x4f, 0x47, 0x5f, 0x50, 0x49, 0x44, 0x10, - 0x09, 0x12, 0x16, 0x0a, 0x12, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, - 0x5f, 0x4b, 0x45, 0x52, 0x4e, 0x45, 0x4c, 0x10, 0x0a, 0x12, 0x1a, 0x0a, 0x16, 0x42, 0x4f, 0x4f, - 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x50, 0x4f, 0x57, 0x45, 0x52, 0x5f, 0x46, - 0x41, 0x49, 0x4c, 0x10, 0x0b, 0x12, 0x17, 0x0a, 0x13, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, - 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x0c, 0x12, 0x1c, - 0x0a, 0x18, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x56, 0x41, - 0x55, 0x4c, 0x54, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x0d, 0x12, 0x1c, 0x0a, 0x18, + 0x4f, 0x4c, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x23, 0x0a, 0x1f, 0x5a, 0x5f, 0x43, + 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f, + 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x51, 0x4d, 0x49, 0x10, 0x01, 0x12, 0x24, + 0x0a, 0x20, 0x5a, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x43, 0x4f, 0x4e, + 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x4d, 0x42, + 0x49, 0x4d, 0x10, 0x02, 0x2a, 0xb1, 0x02, 0x0a, 0x0c, 0x5a, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x18, 0x0a, 0x14, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1b, + 0x0a, 0x17, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, + 0x52, 0x45, 0x42, 0x4f, 0x4f, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x22, 0x0a, 0x1e, 0x5a, + 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x4d, 0x41, 0x49, + 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x10, 0x03, 0x12, + 0x21, 0x0a, 0x1d, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x42, 0x41, 0x53, 0x45, 0x4f, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x49, 0x4e, 0x47, + 0x10, 0x04, 0x12, 0x19, 0x0a, 0x15, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x45, 0x5f, 0x42, 0x4f, 0x4f, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x24, 0x0a, + 0x20, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, + 0x52, 0x45, 0x50, 0x41, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x4f, 0x57, 0x45, 0x52, 0x4f, 0x46, + 0x46, 0x10, 0x06, 0x12, 0x1e, 0x0a, 0x1a, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x4f, 0x57, 0x45, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x4f, 0x46, + 0x46, 0x10, 0x07, 0x12, 0x23, 0x0a, 0x1f, 0x5a, 0x44, 0x45, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x50, 0x52, 0x45, 0x50, 0x41, 0x52, 0x45, 0x44, 0x5f, 0x50, 0x4f, + 0x57, 0x45, 0x52, 0x4f, 0x46, 0x46, 0x10, 0x08, 0x2a, 0xf5, 0x01, 0x0a, 0x0d, 0x53, 0x74, 0x6f, + 0x72, 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x54, + 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, + 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, + 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x4f, 0x4e, 0x4c, + 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x45, 0x47, 0x52, 0x41, 0x44, 0x45, 0x44, + 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x55, 0x53, 0x5f, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x45, 0x44, 0x10, 0x03, 0x12, 0x1a, + 0x0a, 0x16, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x4f, 0x46, 0x46, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x54, + 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x41, + 0x56, 0x41, 0x49, 0x4c, 0x10, 0x05, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, + 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x44, + 0x10, 0x06, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, 0x53, 0x50, 0x45, 0x4e, 0x44, 0x45, 0x44, 0x10, 0x07, + 0x2a, 0xe3, 0x01, 0x0a, 0x0f, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x61, 0x69, 0x64, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, + 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x4f, 0x52, 0x41, + 0x47, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x41, 0x49, + 0x44, 0x30, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, + 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x31, 0x10, + 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x52, 0x41, 0x49, + 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x35, 0x10, 0x03, 0x12, 0x1b, + 0x0a, 0x17, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x36, 0x10, 0x04, 0x12, 0x1b, 0x0a, 0x17, 0x53, + 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x52, 0x41, 0x49, 0x44, 0x37, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x54, 0x4f, 0x52, + 0x41, 0x47, 0x45, 0x5f, 0x52, 0x41, 0x49, 0x44, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x4f, + 0x52, 0x41, 0x49, 0x44, 0x10, 0x06, 0x2a, 0x6b, 0x0a, 0x0f, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, + 0x65, 0x54, 0x79, 0x70, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x54, 0x4f, + 0x52, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, + 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x46, + 0x4f, 0x5f, 0x45, 0x58, 0x54, 0x34, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x4f, 0x52, + 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x5a, 0x46, + 0x53, 0x10, 0x02, 0x2a, 0x85, 0x01, 0x0a, 0x0b, 0x43, 0x50, 0x55, 0x50, 0x6f, 0x6f, 0x6c, 0x4b, + 0x69, 0x6e, 0x64, 0x12, 0x1d, 0x0a, 0x19, 0x43, 0x50, 0x55, 0x5f, 0x50, 0x4f, 0x4f, 0x4c, 0x5f, + 0x4b, 0x49, 0x4e, 0x44, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x43, 0x50, 0x55, 0x5f, 0x50, 0x4f, 0x4f, 0x4c, 0x5f, 0x4b, + 0x49, 0x4e, 0x44, 0x5f, 0x48, 0x4f, 0x55, 0x53, 0x45, 0x4b, 0x45, 0x45, 0x50, 0x49, 0x4e, 0x47, + 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x50, 0x55, 0x5f, 0x50, 0x4f, 0x4f, 0x4c, 0x5f, 0x4b, + 0x49, 0x4e, 0x44, 0x5f, 0x44, 0x45, 0x44, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x10, 0x02, 0x12, + 0x1a, 0x0a, 0x16, 0x43, 0x50, 0x55, 0x5f, 0x50, 0x4f, 0x4f, 0x4c, 0x5f, 0x4b, 0x49, 0x4e, 0x44, + 0x5f, 0x49, 0x53, 0x4f, 0x4c, 0x41, 0x54, 0x45, 0x44, 0x10, 0x03, 0x2a, 0xb0, 0x07, 0x0a, 0x0d, + 0x41, 0x50, 0x49, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x1e, 0x0a, + 0x1a, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, + 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1f, 0x0a, + 0x1b, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, + 0x52, 0x45, 0x54, 0x52, 0x59, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, 0x1b, + 0x0a, 0x17, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, + 0x5f, 0x53, 0x48, 0x55, 0x54, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x02, 0x12, 0x29, 0x0a, 0x25, 0x41, + 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x53, 0x54, + 0x41, 0x52, 0x54, 0x5f, 0x44, 0x45, 0x4c, 0x41, 0x59, 0x5f, 0x49, 0x4e, 0x5f, 0x53, 0x45, 0x43, + 0x4f, 0x4e, 0x44, 0x53, 0x10, 0x03, 0x12, 0x1b, 0x0a, 0x17, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, + 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x45, 0x44, 0x47, 0x45, 0x56, 0x49, 0x45, + 0x57, 0x10, 0x04, 0x12, 0x23, 0x0a, 0x1f, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, + 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x56, 0x4f, 0x4c, 0x55, 0x4d, 0x45, 0x5f, 0x53, 0x4e, 0x41, + 0x50, 0x53, 0x48, 0x4f, 0x54, 0x53, 0x10, 0x05, 0x12, 0x2b, 0x0a, 0x27, 0x41, 0x50, 0x49, 0x5f, + 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, + 0x52, 0x4b, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x52, 0x4f, 0x55, 0x54, + 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x1c, 0x0a, 0x18, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, + 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x4d, 0x4f, 0x44, + 0x45, 0x10, 0x07, 0x12, 0x16, 0x0a, 0x12, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, + 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x4d, 0x54, 0x55, 0x10, 0x08, 0x12, 0x26, 0x0a, 0x22, 0x41, + 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x44, + 0x41, 0x50, 0x54, 0x45, 0x52, 0x5f, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x4c, 0x41, 0x42, 0x45, 0x4c, + 0x53, 0x10, 0x09, 0x12, 0x2f, 0x0a, 0x2b, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, + 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x45, 0x4e, 0x46, 0x4f, 0x52, 0x43, 0x45, 0x44, 0x5f, 0x4e, + 0x45, 0x54, 0x5f, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x46, 0x41, 0x43, 0x45, 0x5f, 0x4f, 0x52, 0x44, + 0x45, 0x52, 0x10, 0x0a, 0x12, 0x1c, 0x0a, 0x18, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, + 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x4e, 0x54, 0x50, 0x53, 0x5f, 0x46, 0x51, 0x44, 0x4e, + 0x10, 0x0b, 0x12, 0x26, 0x0a, 0x22, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, + 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x57, 0x49, 0x4e, 0x5f, 0x4c, 0x49, 0x43, 0x5f, 0x50, 0x41, 0x53, + 0x53, 0x54, 0x48, 0x52, 0x4f, 0x55, 0x47, 0x48, 0x10, 0x0c, 0x12, 0x2d, 0x0a, 0x29, 0x41, 0x50, + 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x56, 0x4f, 0x4c, + 0x55, 0x4d, 0x45, 0x5f, 0x53, 0x4e, 0x41, 0x50, 0x53, 0x48, 0x4f, 0x54, 0x53, 0x5f, 0x49, 0x4d, + 0x4d, 0x45, 0x44, 0x49, 0x41, 0x54, 0x45, 0x10, 0x0d, 0x12, 0x2b, 0x0a, 0x27, 0x41, 0x50, 0x49, + 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x45, 0x4e, 0x43, 0x52, + 0x59, 0x50, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x54, 0x43, 0x48, 0x5f, 0x45, 0x4e, 0x56, 0x45, + 0x4c, 0x4f, 0x50, 0x45, 0x10, 0x0e, 0x12, 0x2a, 0x0a, 0x26, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, + 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x53, 0x49, 0x4e, 0x47, 0x4c, 0x45, 0x5f, + 0x53, 0x54, 0x41, 0x43, 0x4b, 0x5f, 0x49, 0x50, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, + 0x10, 0x0f, 0x12, 0x29, 0x0a, 0x25, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, + 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x5f, 0x41, 0x54, + 0x54, 0x41, 0x43, 0x48, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x10, 0x10, 0x12, 0x2a, 0x0a, + 0x26, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, + 0x45, 0x44, 0x47, 0x45, 0x56, 0x49, 0x45, 0x57, 0x5f, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, + 0x49, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x11, 0x12, 0x1f, 0x0a, 0x1b, 0x41, 0x50, 0x49, + 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x44, 0x49, 0x53, 0x41, + 0x42, 0x4c, 0x45, 0x5f, 0x56, 0x54, 0x50, 0x4d, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x41, 0x50, + 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x4c, 0x4f, 0x43, + 0x5f, 0x52, 0x45, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x43, 0x4f, 0x4c, 0x4c, 0x45, 0x43, 0x54, 0x5f, + 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x13, 0x12, 0x1f, 0x0a, 0x1b, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, + 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x53, 0x4d, 0x41, 0x52, 0x54, 0x5f, 0x52, + 0x45, 0x50, 0x4f, 0x52, 0x54, 0x10, 0x14, 0x12, 0x26, 0x0a, 0x22, 0x41, 0x50, 0x49, 0x5f, 0x43, + 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x52, 0x54, + 0x5f, 0x54, 0x50, 0x4d, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x4c, 0x4f, 0x47, 0x10, 0x15, 0x12, + 0x34, 0x0a, 0x30, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, + 0x59, 0x5f, 0x41, 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4e, + 0x45, 0x54, 0x5f, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x46, 0x41, 0x43, 0x45, 0x5f, 0x43, 0x48, 0x41, + 0x4e, 0x47, 0x45, 0x10, 0x16, 0x12, 0x27, 0x0a, 0x23, 0x41, 0x50, 0x49, 0x5f, 0x43, 0x41, 0x50, + 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x43, 0x50, 0x55, 0x5f, 0x50, 0x4c, 0x41, 0x43, + 0x45, 0x4d, 0x45, 0x4e, 0x54, 0x5f, 0x50, 0x4f, 0x4c, 0x49, 0x43, 0x59, 0x10, 0x17, 0x2a, 0xb9, + 0x03, 0x0a, 0x0a, 0x42, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, + 0x17, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, + 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x42, 0x4f, + 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x46, 0x49, 0x52, 0x53, 0x54, 0x10, + 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, + 0x5f, 0x52, 0x45, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x43, 0x4d, 0x44, 0x10, 0x02, 0x12, 0x16, 0x0a, + 0x12, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x55, 0x50, 0x44, + 0x41, 0x54, 0x45, 0x10, 0x03, 0x12, 0x18, 0x0a, 0x14, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, + 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, 0x12, + 0x1a, 0x0a, 0x16, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x44, + 0x49, 0x53, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x42, + 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x46, 0x41, 0x54, 0x41, 0x4c, + 0x10, 0x06, 0x12, 0x13, 0x0a, 0x0f, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, + 0x4e, 0x5f, 0x4f, 0x4f, 0x4d, 0x10, 0x07, 0x12, 0x1d, 0x0a, 0x19, 0x42, 0x4f, 0x4f, 0x54, 0x5f, + 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x57, 0x41, 0x54, 0x43, 0x48, 0x44, 0x4f, 0x47, 0x5f, + 0x48, 0x55, 0x4e, 0x47, 0x10, 0x08, 0x12, 0x1c, 0x0a, 0x18, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, + 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x57, 0x41, 0x54, 0x43, 0x48, 0x44, 0x4f, 0x47, 0x5f, 0x50, + 0x49, 0x44, 0x10, 0x09, 0x12, 0x16, 0x0a, 0x12, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, + 0x53, 0x4f, 0x4e, 0x5f, 0x4b, 0x45, 0x52, 0x4e, 0x45, 0x4c, 0x10, 0x0a, 0x12, 0x1a, 0x0a, 0x16, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x50, 0x4f, 0x57, 0x45, - 0x52, 0x4f, 0x46, 0x46, 0x5f, 0x43, 0x4d, 0x44, 0x10, 0x0e, 0x12, 0x1b, 0x0a, 0x16, 0x42, 0x4f, - 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x50, 0x41, 0x52, 0x53, 0x45, 0x5f, - 0x46, 0x41, 0x49, 0x4c, 0x10, 0xff, 0x01, 0x2a, 0xd6, 0x02, 0x0a, 0x15, 0x4d, 0x61, 0x69, 0x6e, - 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x61, 0x73, 0x6f, - 0x6e, 0x12, 0x20, 0x0a, 0x1c, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, - 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x4e, 0x4f, 0x4e, - 0x45, 0x10, 0x00, 0x12, 0x2a, 0x0a, 0x26, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, - 0x43, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x55, - 0x53, 0x45, 0x52, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, - 0x2b, 0x0a, 0x27, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, - 0x4f, 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x56, 0x41, 0x55, 0x4c, 0x54, - 0x5f, 0x4c, 0x4f, 0x43, 0x4b, 0x45, 0x44, 0x5f, 0x55, 0x50, 0x10, 0x02, 0x12, 0x2a, 0x0a, 0x26, - 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, - 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x4c, 0x4f, 0x57, 0x5f, 0x44, 0x49, 0x53, 0x4b, - 0x5f, 0x53, 0x50, 0x41, 0x43, 0x45, 0x10, 0x03, 0x12, 0x32, 0x0a, 0x2e, 0x4d, 0x41, 0x49, 0x4e, - 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, - 0x53, 0x4f, 0x4e, 0x5f, 0x54, 0x50, 0x4d, 0x5f, 0x45, 0x4e, 0x43, 0x52, 0x59, 0x50, 0x54, 0x49, - 0x4f, 0x4e, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x04, 0x12, 0x2d, 0x0a, 0x29, - 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, - 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x54, 0x50, 0x4d, 0x5f, 0x51, 0x55, 0x4f, 0x54, - 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x05, 0x12, 0x33, 0x0a, 0x2f, 0x4d, + 0x52, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x10, 0x0b, 0x12, 0x17, 0x0a, 0x13, 0x42, 0x4f, 0x4f, 0x54, + 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, + 0x0c, 0x12, 0x1c, 0x0a, 0x18, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, + 0x5f, 0x56, 0x41, 0x55, 0x4c, 0x54, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x0d, 0x12, + 0x1c, 0x0a, 0x18, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x50, + 0x4f, 0x57, 0x45, 0x52, 0x4f, 0x46, 0x46, 0x5f, 0x43, 0x4d, 0x44, 0x10, 0x0e, 0x12, 0x1b, 0x0a, + 0x16, 0x42, 0x4f, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x50, 0x41, 0x52, + 0x53, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x10, 0xff, 0x01, 0x2a, 0xd6, 0x02, 0x0a, 0x15, 0x4d, + 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x65, + 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x1c, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, + 0x4e, 0x43, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, + 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x2a, 0x0a, 0x26, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, + 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, + 0x4e, 0x5f, 0x55, 0x53, 0x45, 0x52, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x45, 0x44, + 0x10, 0x01, 0x12, 0x2b, 0x0a, 0x27, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, + 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x56, 0x41, + 0x55, 0x4c, 0x54, 0x5f, 0x4c, 0x4f, 0x43, 0x4b, 0x45, 0x44, 0x5f, 0x55, 0x50, 0x10, 0x02, 0x12, + 0x2a, 0x0a, 0x26, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, + 0x4f, 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x4c, 0x4f, 0x57, 0x5f, 0x44, + 0x49, 0x53, 0x4b, 0x5f, 0x53, 0x50, 0x41, 0x43, 0x45, 0x10, 0x03, 0x12, 0x32, 0x0a, 0x2e, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, - 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x44, 0x47, 0x45, 0x5f, 0x4e, 0x4f, 0x44, 0x45, - 0x5f, 0x43, 0x45, 0x52, 0x54, 0x53, 0x5f, 0x52, 0x45, 0x46, 0x55, 0x53, 0x45, 0x44, 0x10, 0x06, - 0x2a, 0xb5, 0x02, 0x0a, 0x10, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x1d, 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, - 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x41, 0x54, 0x54, 0x45, - 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x4e, 0x4f, - 0x4e, 0x43, 0x45, 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x01, 0x12, 0x24, 0x0a, 0x20, 0x41, 0x54, - 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, - 0x54, 0x50, 0x4d, 0x5f, 0x51, 0x55, 0x4f, 0x54, 0x45, 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x02, - 0x12, 0x25, 0x0a, 0x21, 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x50, 0x4d, 0x5f, 0x45, 0x53, 0x43, 0x52, 0x4f, 0x57, - 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x03, 0x12, 0x21, 0x0a, 0x1d, 0x41, 0x54, 0x54, 0x45, 0x53, - 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x54, 0x54, - 0x45, 0x53, 0x54, 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x04, 0x12, 0x28, 0x0a, 0x24, 0x41, 0x54, + 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x54, 0x50, 0x4d, 0x5f, 0x45, 0x4e, 0x43, 0x52, 0x59, + 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x04, 0x12, + 0x2d, 0x0a, 0x29, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, + 0x4f, 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x54, 0x50, 0x4d, 0x5f, 0x51, + 0x55, 0x4f, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x05, 0x12, 0x33, + 0x0a, 0x2f, 0x4d, 0x41, 0x49, 0x4e, 0x54, 0x45, 0x4e, 0x41, 0x4e, 0x43, 0x45, 0x5f, 0x4d, 0x4f, + 0x44, 0x45, 0x5f, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x44, 0x47, 0x45, 0x5f, 0x4e, + 0x4f, 0x44, 0x45, 0x5f, 0x43, 0x45, 0x52, 0x54, 0x53, 0x5f, 0x52, 0x45, 0x46, 0x55, 0x53, 0x45, + 0x44, 0x10, 0x06, 0x2a, 0xb5, 0x02, 0x0a, 0x10, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x1d, 0x41, 0x54, 0x54, 0x45, + 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x41, + 0x54, 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x4e, 0x4f, 0x4e, 0x43, 0x45, 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x01, 0x12, 0x24, 0x0a, + 0x20, 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x54, 0x50, 0x4d, 0x5f, 0x51, 0x55, 0x4f, 0x54, 0x45, 0x5f, 0x57, 0x41, 0x49, + 0x54, 0x10, 0x02, 0x12, 0x25, 0x0a, 0x21, 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, + 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x50, 0x4d, 0x5f, 0x45, 0x53, 0x43, + 0x52, 0x4f, 0x57, 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x03, 0x12, 0x21, 0x0a, 0x1d, 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, - 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x5f, 0x45, 0x53, 0x43, 0x52, 0x4f, 0x57, 0x5f, 0x57, 0x41, - 0x49, 0x54, 0x10, 0x05, 0x12, 0x22, 0x0a, 0x1e, 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x54, 0x41, 0x52, - 0x54, 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x06, 0x12, 0x1e, 0x0a, 0x1a, 0x41, 0x54, 0x54, 0x45, - 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x4f, - 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x07, 0x2a, 0x8b, 0x01, 0x0a, 0x13, 0x41, 0x70, 0x70, - 0x49, 0x6e, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x20, 0x0a, 0x1c, 0x41, 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x4d, 0x45, 0x54, - 0x41, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, - 0x10, 0x00, 0x12, 0x27, 0x0a, 0x23, 0x41, 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x4d, - 0x45, 0x54, 0x41, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4b, 0x55, - 0x42, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x10, 0x01, 0x12, 0x29, 0x0a, 0x25, 0x41, - 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x5f, 0x44, 0x41, 0x54, - 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x55, 0x53, 0x54, 0x4f, 0x4d, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x55, 0x53, 0x10, 0x02, 0x2a, 0x61, 0x0a, 0x0c, 0x57, 0x69, 0x72, 0x65, 0x6c, 0x65, - 0x73, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x57, 0x49, 0x52, 0x45, 0x4c, 0x45, - 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, - 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x57, 0x49, 0x52, 0x45, 0x4c, 0x45, 0x53, - 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x49, 0x46, 0x49, 0x10, 0x01, 0x12, 0x1a, 0x0a, - 0x16, 0x57, 0x49, 0x52, 0x45, 0x4c, 0x45, 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, - 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x10, 0x02, 0x2a, 0x71, 0x0a, 0x0c, 0x42, 0x61, 0x73, - 0x65, 0x4f, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4e, - 0x45, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x49, - 0x4e, 0x47, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, - 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x55, 0x50, 0x44, 0x41, 0x54, - 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, - 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x05, - 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x2a, 0xcb, 0x01, 0x0a, - 0x0f, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x73, 0x53, 0x75, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x12, 0x0a, 0x0e, 0x4e, 0x4f, 0x4e, 0x45, 0x5f, 0x53, 0x55, 0x42, 0x53, 0x54, 0x41, 0x54, - 0x55, 0x53, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, - 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x01, 0x12, 0x15, 0x0a, - 0x11, 0x56, 0x45, 0x52, 0x49, 0x46, 0x59, 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, - 0x53, 0x53, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x49, - 0x4e, 0x49, 0x54, 0x49, 0x41, 0x4c, 0x49, 0x5a, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x14, 0x0a, - 0x10, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x45, 0x42, 0x4f, 0x4f, 0x54, 0x49, 0x4e, - 0x47, 0x10, 0x04, 0x12, 0x12, 0x0a, 0x0e, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x45, - 0x53, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, 0x44, 0x41, 0x54, - 0x45, 0x5f, 0x4e, 0x45, 0x45, 0x44, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x5f, 0x43, 0x4f, 0x4e, 0x46, - 0x49, 0x52, 0x4d, 0x10, 0x06, 0x12, 0x13, 0x0a, 0x0f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, - 0x44, 0x45, 0x46, 0x45, 0x52, 0x52, 0x45, 0x44, 0x10, 0x07, 0x2a, 0x68, 0x0a, 0x0c, 0x53, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4e, - 0x41, 0x50, 0x53, 0x48, 0x4f, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, - 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4e, 0x41, - 0x50, 0x53, 0x48, 0x4f, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x50, 0x50, 0x5f, 0x55, - 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4e, 0x41, 0x50, 0x53, - 0x48, 0x4f, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4d, 0x4d, 0x45, 0x44, 0x49, 0x41, - 0x54, 0x45, 0x10, 0x02, 0x2a, 0xb8, 0x01, 0x0a, 0x16, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x43, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x2a, 0x0a, 0x26, 0x5a, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, - 0x52, 0x5f, 0x4e, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x24, 0x0a, 0x20, 0x5a, - 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x4e, 0x4f, - 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, - 0x01, 0x12, 0x27, 0x0a, 0x23, 0x5a, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x43, 0x4c, 0x55, 0x53, - 0x54, 0x45, 0x52, 0x5f, 0x4e, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x4e, 0x4f, 0x54, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x23, 0x0a, 0x1f, 0x5a, 0x5f, - 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x4e, 0x4f, 0x44, - 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x03, 0x2a, - 0x8f, 0x01, 0x0a, 0x0d, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x56, 0x50, 0x4e, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, - 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x56, 0x50, 0x4e, 0x5f, 0x49, 0x4e, 0x49, 0x54, 0x49, 0x41, - 0x4c, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x56, 0x50, 0x4e, 0x5f, 0x43, 0x4f, 0x4e, 0x4e, 0x45, - 0x43, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x56, 0x50, 0x4e, 0x5f, 0x45, - 0x53, 0x54, 0x41, 0x42, 0x4c, 0x49, 0x53, 0x48, 0x45, 0x44, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, - 0x56, 0x50, 0x4e, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, - 0x0f, 0x0a, 0x0b, 0x56, 0x50, 0x4e, 0x5f, 0x52, 0x45, 0x4b, 0x45, 0x59, 0x45, 0x44, 0x10, 0x05, - 0x12, 0x0f, 0x0a, 0x0b, 0x56, 0x50, 0x4e, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, - 0x0a, 0x2a, 0x85, 0x01, 0x0a, 0x15, 0x5a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x49, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1e, 0x0a, 0x1a, 0x5a, - 0x4e, 0x45, 0x54, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x5a, - 0x4e, 0x45, 0x54, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x49, 0x4e, - 0x49, 0x54, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x5a, 0x4e, 0x45, 0x54, 0x49, 0x4e, 0x53, 0x54, - 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x02, 0x12, - 0x18, 0x0a, 0x14, 0x5a, 0x4e, 0x45, 0x54, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x45, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x2a, 0x9e, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, - 0x63, 0x52, 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x1f, 0x0a, 0x1b, - 0x4c, 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x4c, 0x49, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, - 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1c, 0x0a, - 0x18, 0x4c, 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x4c, 0x49, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, - 0x5f, 0x56, 0x45, 0x52, 0x59, 0x5f, 0x4c, 0x4f, 0x57, 0x10, 0x01, 0x12, 0x17, 0x0a, 0x13, 0x4c, - 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x4c, 0x49, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x4c, - 0x4f, 0x57, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x4c, 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x4c, 0x49, - 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x4d, 0x45, 0x44, 0x49, 0x55, 0x4d, 0x10, 0x03, - 0x12, 0x18, 0x0a, 0x14, 0x4c, 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x4c, 0x49, 0x41, 0x42, 0x49, 0x4c, - 0x49, 0x54, 0x59, 0x5f, 0x48, 0x49, 0x47, 0x48, 0x10, 0x04, 0x42, 0x39, 0x0a, 0x13, 0x6f, 0x72, - 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, 0x69, 0x6e, 0x66, - 0x6f, 0x5a, 0x22, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x66, - 0x2d, 0x65, 0x64, 0x67, 0x65, 0x2f, 0x65, 0x76, 0x65, 0x2d, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x6f, - 0x2f, 0x69, 0x6e, 0x66, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x04, 0x12, 0x28, 0x0a, + 0x24, 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x41, 0x54, 0x54, 0x45, 0x53, 0x54, 0x5f, 0x45, 0x53, 0x43, 0x52, 0x4f, 0x57, + 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x05, 0x12, 0x22, 0x0a, 0x1e, 0x41, 0x54, 0x54, 0x45, 0x53, + 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x45, 0x53, + 0x54, 0x41, 0x52, 0x54, 0x5f, 0x57, 0x41, 0x49, 0x54, 0x10, 0x06, 0x12, 0x1e, 0x0a, 0x1a, 0x41, + 0x54, 0x54, 0x45, 0x53, 0x54, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x07, 0x2a, 0x8b, 0x01, 0x0a, 0x13, + 0x41, 0x70, 0x70, 0x49, 0x6e, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x44, 0x61, 0x74, 0x61, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x1c, 0x41, 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x5f, + 0x4d, 0x45, 0x54, 0x41, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, + 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x27, 0x0a, 0x23, 0x41, 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, + 0x54, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x4b, 0x55, 0x42, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x49, 0x47, 0x10, 0x01, 0x12, 0x29, + 0x0a, 0x25, 0x41, 0x50, 0x50, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x5f, + 0x44, 0x41, 0x54, 0x41, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x55, 0x53, 0x54, 0x4f, 0x4d, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x10, 0x02, 0x2a, 0x61, 0x0a, 0x0c, 0x57, 0x69, 0x72, + 0x65, 0x6c, 0x65, 0x73, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x57, 0x49, 0x52, + 0x45, 0x4c, 0x45, 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, + 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x57, 0x49, 0x52, 0x45, + 0x4c, 0x45, 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x57, 0x49, 0x46, 0x49, 0x10, 0x01, + 0x12, 0x1a, 0x0a, 0x16, 0x57, 0x49, 0x52, 0x45, 0x4c, 0x45, 0x53, 0x53, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x43, 0x45, 0x4c, 0x4c, 0x55, 0x4c, 0x41, 0x52, 0x10, 0x02, 0x2a, 0x71, 0x0a, 0x0c, + 0x42, 0x61, 0x73, 0x65, 0x4f, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x08, 0x0a, 0x04, + 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, + 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x4f, 0x57, 0x4e, 0x4c, + 0x4f, 0x41, 0x44, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x55, 0x50, + 0x44, 0x41, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x50, 0x44, 0x41, + 0x54, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0c, 0x0a, 0x08, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, + 0x4b, 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x06, 0x2a, + 0xcb, 0x01, 0x0a, 0x0f, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x73, 0x53, 0x75, 0x62, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x0e, 0x4e, 0x4f, 0x4e, 0x45, 0x5f, 0x53, 0x55, 0x42, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x44, 0x4f, 0x57, 0x4e, 0x4c, + 0x4f, 0x41, 0x44, 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x01, + 0x12, 0x15, 0x0a, 0x11, 0x56, 0x45, 0x52, 0x49, 0x46, 0x59, 0x5f, 0x49, 0x4e, 0x50, 0x52, 0x4f, + 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x55, 0x50, 0x44, 0x41, 0x54, + 0x45, 0x5f, 0x49, 0x4e, 0x49, 0x54, 0x49, 0x41, 0x4c, 0x49, 0x5a, 0x49, 0x4e, 0x47, 0x10, 0x03, + 0x12, 0x14, 0x0a, 0x10, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x45, 0x42, 0x4f, 0x4f, + 0x54, 0x49, 0x4e, 0x47, 0x10, 0x04, 0x12, 0x12, 0x0a, 0x0e, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, + 0x5f, 0x54, 0x45, 0x53, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x55, 0x50, + 0x44, 0x41, 0x54, 0x45, 0x5f, 0x4e, 0x45, 0x45, 0x44, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x5f, 0x43, + 0x4f, 0x4e, 0x46, 0x49, 0x52, 0x4d, 0x10, 0x06, 0x12, 0x13, 0x0a, 0x0f, 0x55, 0x50, 0x44, 0x41, + 0x54, 0x45, 0x5f, 0x44, 0x45, 0x46, 0x45, 0x52, 0x52, 0x45, 0x44, 0x10, 0x07, 0x2a, 0x68, 0x0a, + 0x0c, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, + 0x19, 0x53, 0x4e, 0x41, 0x50, 0x53, 0x48, 0x4f, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, + 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, + 0x53, 0x4e, 0x41, 0x50, 0x53, 0x48, 0x4f, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x50, + 0x50, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4e, + 0x41, 0x50, 0x53, 0x48, 0x4f, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4d, 0x4d, 0x45, + 0x44, 0x49, 0x41, 0x54, 0x45, 0x10, 0x02, 0x2a, 0xb8, 0x01, 0x0a, 0x16, 0x5a, 0x49, 0x6e, 0x66, + 0x6f, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x2a, 0x0a, 0x26, 0x5a, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x43, 0x4c, 0x55, + 0x53, 0x54, 0x45, 0x52, 0x5f, 0x4e, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x24, + 0x0a, 0x20, 0x5a, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, + 0x5f, 0x4e, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x45, 0x41, + 0x44, 0x59, 0x10, 0x01, 0x12, 0x27, 0x0a, 0x23, 0x5a, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x43, + 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x4e, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, + 0x55, 0x53, 0x5f, 0x4e, 0x4f, 0x54, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x02, 0x12, 0x23, 0x0a, + 0x1f, 0x5a, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, + 0x4e, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x57, 0x4e, + 0x10, 0x03, 0x2a, 0x8f, 0x01, 0x0a, 0x0d, 0x5a, 0x49, 0x6e, 0x66, 0x6f, 0x56, 0x70, 0x6e, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x56, 0x50, 0x4e, 0x5f, 0x49, 0x4e, 0x56, 0x41, + 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x56, 0x50, 0x4e, 0x5f, 0x49, 0x4e, 0x49, + 0x54, 0x49, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x56, 0x50, 0x4e, 0x5f, 0x43, 0x4f, + 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x56, 0x50, + 0x4e, 0x5f, 0x45, 0x53, 0x54, 0x41, 0x42, 0x4c, 0x49, 0x53, 0x48, 0x45, 0x44, 0x10, 0x03, 0x12, + 0x11, 0x0a, 0x0d, 0x56, 0x50, 0x4e, 0x5f, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x45, 0x44, + 0x10, 0x04, 0x12, 0x0f, 0x0a, 0x0b, 0x56, 0x50, 0x4e, 0x5f, 0x52, 0x45, 0x4b, 0x45, 0x59, 0x45, + 0x44, 0x10, 0x05, 0x12, 0x0f, 0x0a, 0x0b, 0x56, 0x50, 0x4e, 0x5f, 0x44, 0x45, 0x4c, 0x45, 0x54, + 0x45, 0x44, 0x10, 0x0a, 0x2a, 0x85, 0x01, 0x0a, 0x15, 0x5a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, + 0x6b, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1e, + 0x0a, 0x1a, 0x5a, 0x4e, 0x45, 0x54, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, + 0x0a, 0x13, 0x5a, 0x4e, 0x45, 0x54, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x49, 0x4e, 0x49, 0x54, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x5a, 0x4e, 0x45, 0x54, 0x49, + 0x4e, 0x53, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, + 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x5a, 0x4e, 0x45, 0x54, 0x49, 0x4e, 0x53, 0x54, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x2a, 0x9e, 0x01, 0x0a, + 0x0e, 0x4c, 0x6f, 0x63, 0x52, 0x65, 0x6c, 0x69, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x12, + 0x1f, 0x0a, 0x1b, 0x4c, 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x4c, 0x49, 0x41, 0x42, 0x49, 0x4c, 0x49, + 0x54, 0x59, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, + 0x12, 0x1c, 0x0a, 0x18, 0x4c, 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x4c, 0x49, 0x41, 0x42, 0x49, 0x4c, + 0x49, 0x54, 0x59, 0x5f, 0x56, 0x45, 0x52, 0x59, 0x5f, 0x4c, 0x4f, 0x57, 0x10, 0x01, 0x12, 0x17, + 0x0a, 0x13, 0x4c, 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x4c, 0x49, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, + 0x59, 0x5f, 0x4c, 0x4f, 0x57, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x4c, 0x4f, 0x43, 0x5f, 0x52, + 0x45, 0x4c, 0x49, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x4d, 0x45, 0x44, 0x49, 0x55, + 0x4d, 0x10, 0x03, 0x12, 0x18, 0x0a, 0x14, 0x4c, 0x4f, 0x43, 0x5f, 0x52, 0x45, 0x4c, 0x49, 0x41, + 0x42, 0x49, 0x4c, 0x49, 0x54, 0x59, 0x5f, 0x48, 0x49, 0x47, 0x48, 0x10, 0x04, 0x42, 0x39, 0x0a, + 0x13, 0x6f, 0x72, 0x67, 0x2e, 0x6c, 0x66, 0x65, 0x64, 0x67, 0x65, 0x2e, 0x65, 0x76, 0x65, 0x2e, + 0x69, 0x6e, 0x66, 0x6f, 0x5a, 0x22, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6c, 0x66, 0x2d, 0x65, 0x64, 0x67, 0x65, 0x2f, 0x65, 0x76, 0x65, 0x2d, 0x61, 0x70, 0x69, + 0x2f, 0x67, 0x6f, 0x2f, 0x69, 0x6e, 0x66, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -10057,8 +10308,8 @@ func file_info_info_proto_rawDescGZIP() []byte { return file_info_info_proto_rawDescData } -var file_info_info_proto_enumTypes = make([]protoimpl.EnumInfo, 26) -var file_info_info_proto_msgTypes = make([]protoimpl.MessageInfo, 67) +var file_info_info_proto_enumTypes = make([]protoimpl.EnumInfo, 27) +var file_info_info_proto_msgTypes = make([]protoimpl.MessageInfo, 68) var file_info_info_proto_goTypes = []interface{}{ (DepMetricItemType)(0), // 0: org.lfedge.eve.info.DepMetricItemType (ZInfoTypes)(0), // 1: org.lfedge.eve.info.ZInfoTypes @@ -10073,286 +10324,290 @@ var file_info_info_proto_goTypes = []interface{}{ (StorageStatus)(0), // 10: org.lfedge.eve.info.StorageStatus (StorageRaidType)(0), // 11: org.lfedge.eve.info.StorageRaidType (StorageTypeInfo)(0), // 12: org.lfedge.eve.info.StorageTypeInfo - (APICapability)(0), // 13: org.lfedge.eve.info.APICapability - (BootReason)(0), // 14: org.lfedge.eve.info.BootReason - (MaintenanceModeReason)(0), // 15: org.lfedge.eve.info.MaintenanceModeReason - (AttestationState)(0), // 16: org.lfedge.eve.info.AttestationState - (AppInstMetaDataType)(0), // 17: org.lfedge.eve.info.AppInstMetaDataType - (WirelessType)(0), // 18: org.lfedge.eve.info.WirelessType - (BaseOsStatus)(0), // 19: org.lfedge.eve.info.BaseOsStatus - (BaseOsSubStatus)(0), // 20: org.lfedge.eve.info.BaseOsSubStatus - (SnapshotType)(0), // 21: org.lfedge.eve.info.SnapshotType - (ZInfoClusterNodeStatus)(0), // 22: org.lfedge.eve.info.ZInfoClusterNodeStatus - (ZInfoVpnState)(0), // 23: org.lfedge.eve.info.ZInfoVpnState - (ZNetworkInstanceState)(0), // 24: org.lfedge.eve.info.ZNetworkInstanceState - (LocReliability)(0), // 25: org.lfedge.eve.info.LocReliability - (*SmartAttr)(nil), // 26: org.lfedge.eve.info.SmartAttr - (*StorageDiskInfo)(nil), // 27: org.lfedge.eve.info.StorageDiskInfo - (*DeprecatedMetricItem)(nil), // 28: org.lfedge.eve.info.deprecatedMetricItem - (*ZmetIPAssignmentEntry)(nil), // 29: org.lfedge.eve.info.ZmetIPAssignmentEntry - (*ZmetVifInfo)(nil), // 30: org.lfedge.eve.info.ZmetVifInfo - (*ZioBundle)(nil), // 31: org.lfedge.eve.info.ZioBundle - (*IoAddresses)(nil), // 32: org.lfedge.eve.info.IoAddresses - (*VfPublishedInfo)(nil), // 33: org.lfedge.eve.info.VfPublishedInfo - (*ZInfoManufacturer)(nil), // 34: org.lfedge.eve.info.ZInfoManufacturer - (*ZInfoNetwork)(nil), // 35: org.lfedge.eve.info.ZInfoNetwork - (*GeoLoc)(nil), // 36: org.lfedge.eve.info.GeoLoc - (*ZInfoDNS)(nil), // 37: org.lfedge.eve.info.ZInfoDNS - (*ZInfoSW)(nil), // 38: org.lfedge.eve.info.ZInfoSW - (*VaultInfo)(nil), // 39: org.lfedge.eve.info.VaultInfo - (*DataSecAtRest)(nil), // 40: org.lfedge.eve.info.DataSecAtRest - (*SecurityInfo)(nil), // 41: org.lfedge.eve.info.SecurityInfo - (*ZInfoConfigItem)(nil), // 42: org.lfedge.eve.info.ZInfoConfigItem - (*ZInfoConfigItemStatus)(nil), // 43: org.lfedge.eve.info.ZInfoConfigItemStatus - (*ZInfoAppInstance)(nil), // 44: org.lfedge.eve.info.ZInfoAppInstance - (*ZInfoDeviceTasks)(nil), // 45: org.lfedge.eve.info.ZInfoDeviceTasks - (*ZSimcardInfo)(nil), // 46: org.lfedge.eve.info.ZSimcardInfo - (*ZCellularModuleInfo)(nil), // 47: org.lfedge.eve.info.ZCellularModuleInfo - (*ZCellularProvider)(nil), // 48: org.lfedge.eve.info.ZCellularProvider - (*CellularBearer)(nil), // 49: org.lfedge.eve.info.CellularBearer - (*CellularProfile)(nil), // 50: org.lfedge.eve.info.CellularProfile - (*StorageDiskState)(nil), // 51: org.lfedge.eve.info.StorageDiskState - (*StorageChildren)(nil), // 52: org.lfedge.eve.info.StorageChildren - (*StorageInfo)(nil), // 53: org.lfedge.eve.info.StorageInfo - (*ZInfoDevice)(nil), // 54: org.lfedge.eve.info.ZInfoDevice - (*OptionalCapabilities)(nil), // 55: org.lfedge.eve.info.OptionalCapabilities - (*AttestationInfo)(nil), // 56: org.lfedge.eve.info.AttestationInfo - (*SystemAdapterInfo)(nil), // 57: org.lfedge.eve.info.SystemAdapterInfo - (*DevicePortStatus)(nil), // 58: org.lfedge.eve.info.DevicePortStatus - (*DevicePort)(nil), // 59: org.lfedge.eve.info.DevicePort - (*ProxyStatus)(nil), // 60: org.lfedge.eve.info.ProxyStatus - (*ProxyEntry)(nil), // 61: org.lfedge.eve.info.ProxyEntry - (*WirelessStatus)(nil), // 62: org.lfedge.eve.info.WirelessStatus - (*ZCellularStatus)(nil), // 63: org.lfedge.eve.info.ZCellularStatus - (*ZInfoDevSW)(nil), // 64: org.lfedge.eve.info.ZInfoDevSW - (*ZInfoStorage)(nil), // 65: org.lfedge.eve.info.ZInfoStorage - (*ZInfoSnapshot)(nil), // 66: org.lfedge.eve.info.ZInfoSnapshot - (*ZInfoClusterNode)(nil), // 67: org.lfedge.eve.info.ZInfoClusterNode - (*ZInfoApp)(nil), // 68: org.lfedge.eve.info.ZInfoApp - (*ZInfoVpnLinkInfo)(nil), // 69: org.lfedge.eve.info.ZInfoVpnLinkInfo - (*ZInfoVpnLink)(nil), // 70: org.lfedge.eve.info.ZInfoVpnLink - (*ZInfoVpnEndPoint)(nil), // 71: org.lfedge.eve.info.ZInfoVpnEndPoint - (*ZInfoVpnConn)(nil), // 72: org.lfedge.eve.info.ZInfoVpnConn - (*ZInfoVpn)(nil), // 73: org.lfedge.eve.info.ZInfoVpn - (*ZInfoNetworkInstance)(nil), // 74: org.lfedge.eve.info.ZInfoNetworkInstance - (*IPRoute)(nil), // 75: org.lfedge.eve.info.IPRoute - (*UsageInfo)(nil), // 76: org.lfedge.eve.info.UsageInfo - (*VolumeResources)(nil), // 77: org.lfedge.eve.info.VolumeResources - (*ZInfoVolume)(nil), // 78: org.lfedge.eve.info.ZInfoVolume - (*ContentResources)(nil), // 79: org.lfedge.eve.info.ContentResources - (*ZInfoContentTree)(nil), // 80: org.lfedge.eve.info.ZInfoContentTree - (*ZInfoBlob)(nil), // 81: org.lfedge.eve.info.ZInfoBlob - (*ZInfoBlobList)(nil), // 82: org.lfedge.eve.info.ZInfoBlobList - (*ZInfoMsg)(nil), // 83: org.lfedge.eve.info.ZInfoMsg - (*Capabilities)(nil), // 84: org.lfedge.eve.info.Capabilities - (*ZInfoAppInstMetaData)(nil), // 85: org.lfedge.eve.info.ZInfoAppInstMetaData - (*ZInfoEdgeview)(nil), // 86: org.lfedge.eve.info.ZInfoEdgeview - (*ZInfoLocation)(nil), // 87: org.lfedge.eve.info.ZInfoLocation - (*ZInfoKubeClusterUpdateStatus)(nil), // 88: org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus - (*ZInfoKubeCluster)(nil), // 89: org.lfedge.eve.info.ZInfoKubeCluster - (*ZInfoHardware)(nil), // 90: org.lfedge.eve.info.ZInfoHardware - nil, // 91: org.lfedge.eve.info.ZInfoConfigItemStatus.ConfigItemsEntry - nil, // 92: org.lfedge.eve.info.ZInfoConfigItemStatus.UnknownConfigItemsEntry - (evecommon.PhyIoType)(0), // 93: org.lfedge.eve.common.PhyIoType - (evecommon.PhyIoMemberUsage)(0), // 94: org.lfedge.eve.common.PhyIoMemberUsage - (*ErrorInfo)(nil), // 95: org.lfedge.eve.info.ErrorInfo - (evecommon.BearerType)(0), // 96: org.lfedge.eve.common.BearerType - (evecommon.CellularIPType)(0), // 97: org.lfedge.eve.common.CellularIPType - (*timestamppb.Timestamp)(nil), // 98: google.protobuf.Timestamp - (*evecommon.DiskDescription)(nil), // 99: org.lfedge.eve.common.DiskDescription - (*CertInfo)(nil), // 100: org.lfedge.eve.info.CertInfo - (*evecommon.PortConfigSource)(nil), // 101: org.lfedge.eve.common.PortConfigSource - (*PNACStatus)(nil), // 102: org.lfedge.eve.info.PNACStatus - (*BondStatus)(nil), // 103: org.lfedge.eve.info.BondStatus - (evecommon.RadioAccessTechnology)(0), // 104: org.lfedge.eve.common.RadioAccessTechnology - (*ZInfoPatchEnvelope)(nil), // 105: org.lfedge.eve.info.ZInfoPatchEnvelope - (*ZInfoNTPSources)(nil), // 106: org.lfedge.eve.info.ZInfoNTPSources - (KubeComp)(0), // 107: org.lfedge.eve.info.KubeComp - (KubeCompUpdateStatus)(0), // 108: org.lfedge.eve.info.KubeCompUpdateStatus - (*KubeNodeInfo)(nil), // 109: org.lfedge.eve.info.KubeNodeInfo - (*KubePodNameSpaceInfo)(nil), // 110: org.lfedge.eve.info.KubePodNameSpaceInfo - (*KubeEVEAppPodInfo)(nil), // 111: org.lfedge.eve.info.KubeEVEAppPodInfo - (*KubeStorageInfo)(nil), // 112: org.lfedge.eve.info.KubeStorageInfo - (*KubeVMIInfo)(nil), // 113: org.lfedge.eve.info.KubeVMIInfo - (*HardwareInventory)(nil), // 114: org.lfedge.eve.info.HardwareInventory + (CPUPoolKind)(0), // 13: org.lfedge.eve.info.CPUPoolKind + (APICapability)(0), // 14: org.lfedge.eve.info.APICapability + (BootReason)(0), // 15: org.lfedge.eve.info.BootReason + (MaintenanceModeReason)(0), // 16: org.lfedge.eve.info.MaintenanceModeReason + (AttestationState)(0), // 17: org.lfedge.eve.info.AttestationState + (AppInstMetaDataType)(0), // 18: org.lfedge.eve.info.AppInstMetaDataType + (WirelessType)(0), // 19: org.lfedge.eve.info.WirelessType + (BaseOsStatus)(0), // 20: org.lfedge.eve.info.BaseOsStatus + (BaseOsSubStatus)(0), // 21: org.lfedge.eve.info.BaseOsSubStatus + (SnapshotType)(0), // 22: org.lfedge.eve.info.SnapshotType + (ZInfoClusterNodeStatus)(0), // 23: org.lfedge.eve.info.ZInfoClusterNodeStatus + (ZInfoVpnState)(0), // 24: org.lfedge.eve.info.ZInfoVpnState + (ZNetworkInstanceState)(0), // 25: org.lfedge.eve.info.ZNetworkInstanceState + (LocReliability)(0), // 26: org.lfedge.eve.info.LocReliability + (*SmartAttr)(nil), // 27: org.lfedge.eve.info.SmartAttr + (*StorageDiskInfo)(nil), // 28: org.lfedge.eve.info.StorageDiskInfo + (*DeprecatedMetricItem)(nil), // 29: org.lfedge.eve.info.deprecatedMetricItem + (*ZmetIPAssignmentEntry)(nil), // 30: org.lfedge.eve.info.ZmetIPAssignmentEntry + (*ZmetVifInfo)(nil), // 31: org.lfedge.eve.info.ZmetVifInfo + (*ZioBundle)(nil), // 32: org.lfedge.eve.info.ZioBundle + (*IoAddresses)(nil), // 33: org.lfedge.eve.info.IoAddresses + (*VfPublishedInfo)(nil), // 34: org.lfedge.eve.info.VfPublishedInfo + (*ZInfoManufacturer)(nil), // 35: org.lfedge.eve.info.ZInfoManufacturer + (*ZInfoNetwork)(nil), // 36: org.lfedge.eve.info.ZInfoNetwork + (*GeoLoc)(nil), // 37: org.lfedge.eve.info.GeoLoc + (*ZInfoDNS)(nil), // 38: org.lfedge.eve.info.ZInfoDNS + (*ZInfoSW)(nil), // 39: org.lfedge.eve.info.ZInfoSW + (*VaultInfo)(nil), // 40: org.lfedge.eve.info.VaultInfo + (*DataSecAtRest)(nil), // 41: org.lfedge.eve.info.DataSecAtRest + (*SecurityInfo)(nil), // 42: org.lfedge.eve.info.SecurityInfo + (*ZInfoConfigItem)(nil), // 43: org.lfedge.eve.info.ZInfoConfigItem + (*ZInfoConfigItemStatus)(nil), // 44: org.lfedge.eve.info.ZInfoConfigItemStatus + (*ZInfoAppInstance)(nil), // 45: org.lfedge.eve.info.ZInfoAppInstance + (*ZInfoDeviceTasks)(nil), // 46: org.lfedge.eve.info.ZInfoDeviceTasks + (*ZSimcardInfo)(nil), // 47: org.lfedge.eve.info.ZSimcardInfo + (*ZCellularModuleInfo)(nil), // 48: org.lfedge.eve.info.ZCellularModuleInfo + (*ZCellularProvider)(nil), // 49: org.lfedge.eve.info.ZCellularProvider + (*CellularBearer)(nil), // 50: org.lfedge.eve.info.CellularBearer + (*CellularProfile)(nil), // 51: org.lfedge.eve.info.CellularProfile + (*StorageDiskState)(nil), // 52: org.lfedge.eve.info.StorageDiskState + (*StorageChildren)(nil), // 53: org.lfedge.eve.info.StorageChildren + (*StorageInfo)(nil), // 54: org.lfedge.eve.info.StorageInfo + (*CPUPoolUtilization)(nil), // 55: org.lfedge.eve.info.CPUPoolUtilization + (*ZInfoDevice)(nil), // 56: org.lfedge.eve.info.ZInfoDevice + (*OptionalCapabilities)(nil), // 57: org.lfedge.eve.info.OptionalCapabilities + (*AttestationInfo)(nil), // 58: org.lfedge.eve.info.AttestationInfo + (*SystemAdapterInfo)(nil), // 59: org.lfedge.eve.info.SystemAdapterInfo + (*DevicePortStatus)(nil), // 60: org.lfedge.eve.info.DevicePortStatus + (*DevicePort)(nil), // 61: org.lfedge.eve.info.DevicePort + (*ProxyStatus)(nil), // 62: org.lfedge.eve.info.ProxyStatus + (*ProxyEntry)(nil), // 63: org.lfedge.eve.info.ProxyEntry + (*WirelessStatus)(nil), // 64: org.lfedge.eve.info.WirelessStatus + (*ZCellularStatus)(nil), // 65: org.lfedge.eve.info.ZCellularStatus + (*ZInfoDevSW)(nil), // 66: org.lfedge.eve.info.ZInfoDevSW + (*ZInfoStorage)(nil), // 67: org.lfedge.eve.info.ZInfoStorage + (*ZInfoSnapshot)(nil), // 68: org.lfedge.eve.info.ZInfoSnapshot + (*ZInfoClusterNode)(nil), // 69: org.lfedge.eve.info.ZInfoClusterNode + (*ZInfoApp)(nil), // 70: org.lfedge.eve.info.ZInfoApp + (*ZInfoVpnLinkInfo)(nil), // 71: org.lfedge.eve.info.ZInfoVpnLinkInfo + (*ZInfoVpnLink)(nil), // 72: org.lfedge.eve.info.ZInfoVpnLink + (*ZInfoVpnEndPoint)(nil), // 73: org.lfedge.eve.info.ZInfoVpnEndPoint + (*ZInfoVpnConn)(nil), // 74: org.lfedge.eve.info.ZInfoVpnConn + (*ZInfoVpn)(nil), // 75: org.lfedge.eve.info.ZInfoVpn + (*ZInfoNetworkInstance)(nil), // 76: org.lfedge.eve.info.ZInfoNetworkInstance + (*IPRoute)(nil), // 77: org.lfedge.eve.info.IPRoute + (*UsageInfo)(nil), // 78: org.lfedge.eve.info.UsageInfo + (*VolumeResources)(nil), // 79: org.lfedge.eve.info.VolumeResources + (*ZInfoVolume)(nil), // 80: org.lfedge.eve.info.ZInfoVolume + (*ContentResources)(nil), // 81: org.lfedge.eve.info.ContentResources + (*ZInfoContentTree)(nil), // 82: org.lfedge.eve.info.ZInfoContentTree + (*ZInfoBlob)(nil), // 83: org.lfedge.eve.info.ZInfoBlob + (*ZInfoBlobList)(nil), // 84: org.lfedge.eve.info.ZInfoBlobList + (*ZInfoMsg)(nil), // 85: org.lfedge.eve.info.ZInfoMsg + (*Capabilities)(nil), // 86: org.lfedge.eve.info.Capabilities + (*ZInfoAppInstMetaData)(nil), // 87: org.lfedge.eve.info.ZInfoAppInstMetaData + (*ZInfoEdgeview)(nil), // 88: org.lfedge.eve.info.ZInfoEdgeview + (*ZInfoLocation)(nil), // 89: org.lfedge.eve.info.ZInfoLocation + (*ZInfoKubeClusterUpdateStatus)(nil), // 90: org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus + (*ZInfoKubeCluster)(nil), // 91: org.lfedge.eve.info.ZInfoKubeCluster + (*ZInfoHardware)(nil), // 92: org.lfedge.eve.info.ZInfoHardware + nil, // 93: org.lfedge.eve.info.ZInfoConfigItemStatus.ConfigItemsEntry + nil, // 94: org.lfedge.eve.info.ZInfoConfigItemStatus.UnknownConfigItemsEntry + (evecommon.PhyIoType)(0), // 95: org.lfedge.eve.common.PhyIoType + (evecommon.PhyIoMemberUsage)(0), // 96: org.lfedge.eve.common.PhyIoMemberUsage + (*ErrorInfo)(nil), // 97: org.lfedge.eve.info.ErrorInfo + (evecommon.BearerType)(0), // 98: org.lfedge.eve.common.BearerType + (evecommon.CellularIPType)(0), // 99: org.lfedge.eve.common.CellularIPType + (*timestamppb.Timestamp)(nil), // 100: google.protobuf.Timestamp + (*evecommon.DiskDescription)(nil), // 101: org.lfedge.eve.common.DiskDescription + (*CertInfo)(nil), // 102: org.lfedge.eve.info.CertInfo + (*evecommon.PortConfigSource)(nil), // 103: org.lfedge.eve.common.PortConfigSource + (*PNACStatus)(nil), // 104: org.lfedge.eve.info.PNACStatus + (*BondStatus)(nil), // 105: org.lfedge.eve.info.BondStatus + (evecommon.RadioAccessTechnology)(0), // 106: org.lfedge.eve.common.RadioAccessTechnology + (*ZInfoPatchEnvelope)(nil), // 107: org.lfedge.eve.info.ZInfoPatchEnvelope + (*ZInfoNTPSources)(nil), // 108: org.lfedge.eve.info.ZInfoNTPSources + (KubeComp)(0), // 109: org.lfedge.eve.info.KubeComp + (KubeCompUpdateStatus)(0), // 110: org.lfedge.eve.info.KubeCompUpdateStatus + (*KubeNodeInfo)(nil), // 111: org.lfedge.eve.info.KubeNodeInfo + (*KubePodNameSpaceInfo)(nil), // 112: org.lfedge.eve.info.KubePodNameSpaceInfo + (*KubeEVEAppPodInfo)(nil), // 113: org.lfedge.eve.info.KubeEVEAppPodInfo + (*KubeStorageInfo)(nil), // 114: org.lfedge.eve.info.KubeStorageInfo + (*KubeVMIInfo)(nil), // 115: org.lfedge.eve.info.KubeVMIInfo + (*HardwareInventory)(nil), // 116: org.lfedge.eve.info.HardwareInventory } var file_info_info_proto_depIdxs = []int32{ - 26, // 0: org.lfedge.eve.info.StorageDiskInfo.smart_attr:type_name -> org.lfedge.eve.info.SmartAttr + 27, // 0: org.lfedge.eve.info.StorageDiskInfo.smart_attr:type_name -> org.lfedge.eve.info.SmartAttr 0, // 1: org.lfedge.eve.info.deprecatedMetricItem.type:type_name -> org.lfedge.eve.info.DepMetricItemType - 93, // 2: org.lfedge.eve.info.ZioBundle.type:type_name -> org.lfedge.eve.common.PhyIoType - 32, // 3: org.lfedge.eve.info.ZioBundle.ioAddressList:type_name -> org.lfedge.eve.info.IoAddresses - 94, // 4: org.lfedge.eve.info.ZioBundle.usage:type_name -> org.lfedge.eve.common.PhyIoMemberUsage - 95, // 5: org.lfedge.eve.info.ZioBundle.err:type_name -> org.lfedge.eve.info.ErrorInfo - 33, // 6: org.lfedge.eve.info.IoAddresses.vf_info:type_name -> org.lfedge.eve.info.VfPublishedInfo - 37, // 7: org.lfedge.eve.info.ZInfoNetwork.dns:type_name -> org.lfedge.eve.info.ZInfoDNS - 36, // 8: org.lfedge.eve.info.ZInfoNetwork.location:type_name -> org.lfedge.eve.info.GeoLoc - 95, // 9: org.lfedge.eve.info.ZInfoNetwork.networkErr:type_name -> org.lfedge.eve.info.ErrorInfo - 60, // 10: org.lfedge.eve.info.ZInfoNetwork.proxy:type_name -> org.lfedge.eve.info.ProxyStatus + 95, // 2: org.lfedge.eve.info.ZioBundle.type:type_name -> org.lfedge.eve.common.PhyIoType + 33, // 3: org.lfedge.eve.info.ZioBundle.ioAddressList:type_name -> org.lfedge.eve.info.IoAddresses + 96, // 4: org.lfedge.eve.info.ZioBundle.usage:type_name -> org.lfedge.eve.common.PhyIoMemberUsage + 97, // 5: org.lfedge.eve.info.ZioBundle.err:type_name -> org.lfedge.eve.info.ErrorInfo + 34, // 6: org.lfedge.eve.info.IoAddresses.vf_info:type_name -> org.lfedge.eve.info.VfPublishedInfo + 38, // 7: org.lfedge.eve.info.ZInfoNetwork.dns:type_name -> org.lfedge.eve.info.ZInfoDNS + 37, // 8: org.lfedge.eve.info.ZInfoNetwork.location:type_name -> org.lfedge.eve.info.GeoLoc + 97, // 9: org.lfedge.eve.info.ZInfoNetwork.networkErr:type_name -> org.lfedge.eve.info.ErrorInfo + 62, // 10: org.lfedge.eve.info.ZInfoNetwork.proxy:type_name -> org.lfedge.eve.info.ProxyStatus 2, // 11: org.lfedge.eve.info.ZInfoSW.state:type_name -> org.lfedge.eve.info.ZSwState 4, // 12: org.lfedge.eve.info.VaultInfo.status:type_name -> org.lfedge.eve.info.DataSecAtRestStatus - 95, // 13: org.lfedge.eve.info.VaultInfo.vaultErr:type_name -> org.lfedge.eve.info.ErrorInfo + 97, // 13: org.lfedge.eve.info.VaultInfo.vaultErr:type_name -> org.lfedge.eve.info.ErrorInfo 5, // 14: org.lfedge.eve.info.VaultInfo.pcrStatus:type_name -> org.lfedge.eve.info.PCRStatus 4, // 15: org.lfedge.eve.info.DataSecAtRest.status:type_name -> org.lfedge.eve.info.DataSecAtRestStatus - 39, // 16: org.lfedge.eve.info.DataSecAtRest.vaultList:type_name -> org.lfedge.eve.info.VaultInfo - 91, // 17: org.lfedge.eve.info.ZInfoConfigItemStatus.configItems:type_name -> org.lfedge.eve.info.ZInfoConfigItemStatus.ConfigItemsEntry - 92, // 18: org.lfedge.eve.info.ZInfoConfigItemStatus.unknownConfigItems:type_name -> org.lfedge.eve.info.ZInfoConfigItemStatus.UnknownConfigItemsEntry + 40, // 16: org.lfedge.eve.info.DataSecAtRest.vaultList:type_name -> org.lfedge.eve.info.VaultInfo + 93, // 17: org.lfedge.eve.info.ZInfoConfigItemStatus.configItems:type_name -> org.lfedge.eve.info.ZInfoConfigItemStatus.ConfigItemsEntry + 94, // 18: org.lfedge.eve.info.ZInfoConfigItemStatus.unknownConfigItems:type_name -> org.lfedge.eve.info.ZInfoConfigItemStatus.UnknownConfigItemsEntry 6, // 19: org.lfedge.eve.info.ZSimcardInfo.type:type_name -> org.lfedge.eve.info.SimType 7, // 20: org.lfedge.eve.info.ZCellularModuleInfo.operating_state:type_name -> org.lfedge.eve.info.ZCellularOperatingState 8, // 21: org.lfedge.eve.info.ZCellularModuleInfo.control_protocol:type_name -> org.lfedge.eve.info.ZCellularControlProtocol - 96, // 22: org.lfedge.eve.info.CellularBearer.bearer_type:type_name -> org.lfedge.eve.common.BearerType - 97, // 23: org.lfedge.eve.info.CellularBearer.ip_type:type_name -> org.lfedge.eve.common.CellularIPType - 98, // 24: org.lfedge.eve.info.CellularBearer.connected_at:type_name -> google.protobuf.Timestamp - 96, // 25: org.lfedge.eve.info.CellularProfile.bearer_type:type_name -> org.lfedge.eve.common.BearerType - 97, // 26: org.lfedge.eve.info.CellularProfile.ip_type:type_name -> org.lfedge.eve.common.CellularIPType - 99, // 27: org.lfedge.eve.info.StorageDiskState.disk_name:type_name -> org.lfedge.eve.common.DiskDescription + 98, // 22: org.lfedge.eve.info.CellularBearer.bearer_type:type_name -> org.lfedge.eve.common.BearerType + 99, // 23: org.lfedge.eve.info.CellularBearer.ip_type:type_name -> org.lfedge.eve.common.CellularIPType + 100, // 24: org.lfedge.eve.info.CellularBearer.connected_at:type_name -> google.protobuf.Timestamp + 98, // 25: org.lfedge.eve.info.CellularProfile.bearer_type:type_name -> org.lfedge.eve.common.BearerType + 99, // 26: org.lfedge.eve.info.CellularProfile.ip_type:type_name -> org.lfedge.eve.common.CellularIPType + 101, // 27: org.lfedge.eve.info.StorageDiskState.disk_name:type_name -> org.lfedge.eve.common.DiskDescription 10, // 28: org.lfedge.eve.info.StorageDiskState.status:type_name -> org.lfedge.eve.info.StorageStatus 11, // 29: org.lfedge.eve.info.StorageChildren.current_raid:type_name -> org.lfedge.eve.info.StorageRaidType - 51, // 30: org.lfedge.eve.info.StorageChildren.disks:type_name -> org.lfedge.eve.info.StorageDiskState - 52, // 31: org.lfedge.eve.info.StorageChildren.children:type_name -> org.lfedge.eve.info.StorageChildren + 52, // 30: org.lfedge.eve.info.StorageChildren.disks:type_name -> org.lfedge.eve.info.StorageDiskState + 53, // 31: org.lfedge.eve.info.StorageChildren.children:type_name -> org.lfedge.eve.info.StorageChildren 12, // 32: org.lfedge.eve.info.StorageInfo.storage_type:type_name -> org.lfedge.eve.info.StorageTypeInfo 11, // 33: org.lfedge.eve.info.StorageInfo.current_raid:type_name -> org.lfedge.eve.info.StorageRaidType 10, // 34: org.lfedge.eve.info.StorageInfo.storage_state:type_name -> org.lfedge.eve.info.StorageStatus - 51, // 35: org.lfedge.eve.info.StorageInfo.disks:type_name -> org.lfedge.eve.info.StorageDiskState - 52, // 36: org.lfedge.eve.info.StorageInfo.children:type_name -> org.lfedge.eve.info.StorageChildren - 34, // 37: org.lfedge.eve.info.ZInfoDevice.minfo:type_name -> org.lfedge.eve.info.ZInfoManufacturer - 35, // 38: org.lfedge.eve.info.ZInfoDevice.network:type_name -> org.lfedge.eve.info.ZInfoNetwork - 31, // 39: org.lfedge.eve.info.ZInfoDevice.assignableAdapters:type_name -> org.lfedge.eve.info.ZioBundle - 37, // 40: org.lfedge.eve.info.ZInfoDevice.dns:type_name -> org.lfedge.eve.info.ZInfoDNS - 65, // 41: org.lfedge.eve.info.ZInfoDevice.storageList:type_name -> org.lfedge.eve.info.ZInfoStorage - 98, // 42: org.lfedge.eve.info.ZInfoDevice.bootTime:type_name -> google.protobuf.Timestamp - 64, // 43: org.lfedge.eve.info.ZInfoDevice.swList:type_name -> org.lfedge.eve.info.ZInfoDevSW - 28, // 44: org.lfedge.eve.info.ZInfoDevice.metricItems:type_name -> org.lfedge.eve.info.deprecatedMetricItem - 98, // 45: org.lfedge.eve.info.ZInfoDevice.lastRebootTime:type_name -> google.protobuf.Timestamp - 57, // 46: org.lfedge.eve.info.ZInfoDevice.systemAdapter:type_name -> org.lfedge.eve.info.SystemAdapterInfo - 3, // 47: org.lfedge.eve.info.ZInfoDevice.HSMStatus:type_name -> org.lfedge.eve.info.HwSecurityModuleStatus - 40, // 48: org.lfedge.eve.info.ZInfoDevice.dataSecAtRestInfo:type_name -> org.lfedge.eve.info.DataSecAtRest - 41, // 49: org.lfedge.eve.info.ZInfoDevice.sec_info:type_name -> org.lfedge.eve.info.SecurityInfo - 43, // 50: org.lfedge.eve.info.ZInfoDevice.configItemStatus:type_name -> org.lfedge.eve.info.ZInfoConfigItemStatus - 44, // 51: org.lfedge.eve.info.ZInfoDevice.appInstances:type_name -> org.lfedge.eve.info.ZInfoAppInstance - 14, // 52: org.lfedge.eve.info.ZInfoDevice.last_boot_reason:type_name -> org.lfedge.eve.info.BootReason - 47, // 53: org.lfedge.eve.info.ZInfoDevice.cell_radios:type_name -> org.lfedge.eve.info.ZCellularModuleInfo - 46, // 54: org.lfedge.eve.info.ZInfoDevice.sims:type_name -> org.lfedge.eve.info.ZSimcardInfo - 45, // 55: org.lfedge.eve.info.ZInfoDevice.tasks:type_name -> org.lfedge.eve.info.ZInfoDeviceTasks - 15, // 56: org.lfedge.eve.info.ZInfoDevice.maintenance_mode_reason:type_name -> org.lfedge.eve.info.MaintenanceModeReason - 84, // 57: org.lfedge.eve.info.ZInfoDevice.capabilities:type_name -> org.lfedge.eve.info.Capabilities - 9, // 58: org.lfedge.eve.info.ZInfoDevice.state:type_name -> org.lfedge.eve.info.ZDeviceState - 15, // 59: org.lfedge.eve.info.ZInfoDevice.maintenance_mode_reasons:type_name -> org.lfedge.eve.info.MaintenanceModeReason - 53, // 60: org.lfedge.eve.info.ZInfoDevice.storage_info:type_name -> org.lfedge.eve.info.StorageInfo - 56, // 61: org.lfedge.eve.info.ZInfoDevice.attestation_info:type_name -> org.lfedge.eve.info.AttestationInfo - 13, // 62: org.lfedge.eve.info.ZInfoDevice.api_capability:type_name -> org.lfedge.eve.info.APICapability - 55, // 63: org.lfedge.eve.info.ZInfoDevice.optional_capabilities:type_name -> org.lfedge.eve.info.OptionalCapabilities - 100, // 64: org.lfedge.eve.info.ZInfoDevice.enrolled_certs:type_name -> org.lfedge.eve.info.CertInfo - 16, // 65: org.lfedge.eve.info.AttestationInfo.state:type_name -> org.lfedge.eve.info.AttestationState - 95, // 66: org.lfedge.eve.info.AttestationInfo.error:type_name -> org.lfedge.eve.info.ErrorInfo - 58, // 67: org.lfedge.eve.info.SystemAdapterInfo.status:type_name -> org.lfedge.eve.info.DevicePortStatus - 98, // 68: org.lfedge.eve.info.DevicePortStatus.timePriority:type_name -> google.protobuf.Timestamp - 98, // 69: org.lfedge.eve.info.DevicePortStatus.lastFailed:type_name -> google.protobuf.Timestamp - 98, // 70: org.lfedge.eve.info.DevicePortStatus.lastSucceeded:type_name -> google.protobuf.Timestamp - 59, // 71: org.lfedge.eve.info.DevicePortStatus.ports:type_name -> org.lfedge.eve.info.DevicePort - 60, // 72: org.lfedge.eve.info.DevicePort.proxy:type_name -> org.lfedge.eve.info.ProxyStatus - 37, // 73: org.lfedge.eve.info.DevicePort.dns:type_name -> org.lfedge.eve.info.ZInfoDNS - 36, // 74: org.lfedge.eve.info.DevicePort.location:type_name -> org.lfedge.eve.info.GeoLoc - 95, // 75: org.lfedge.eve.info.DevicePort.err:type_name -> org.lfedge.eve.info.ErrorInfo - 94, // 76: org.lfedge.eve.info.DevicePort.usage:type_name -> org.lfedge.eve.common.PhyIoMemberUsage - 62, // 77: org.lfedge.eve.info.DevicePort.wireless_status:type_name -> org.lfedge.eve.info.WirelessStatus - 101, // 78: org.lfedge.eve.info.DevicePort.config_source:type_name -> org.lfedge.eve.common.PortConfigSource - 102, // 79: org.lfedge.eve.info.DevicePort.pnac_status:type_name -> org.lfedge.eve.info.PNACStatus - 103, // 80: org.lfedge.eve.info.DevicePort.bond_status:type_name -> org.lfedge.eve.info.BondStatus - 61, // 81: org.lfedge.eve.info.ProxyStatus.proxies:type_name -> org.lfedge.eve.info.ProxyEntry - 18, // 82: org.lfedge.eve.info.WirelessStatus.type:type_name -> org.lfedge.eve.info.WirelessType - 63, // 83: org.lfedge.eve.info.WirelessStatus.cellular:type_name -> org.lfedge.eve.info.ZCellularStatus - 48, // 84: org.lfedge.eve.info.ZCellularStatus.providers:type_name -> org.lfedge.eve.info.ZCellularProvider - 104, // 85: org.lfedge.eve.info.ZCellularStatus.current_rats:type_name -> org.lfedge.eve.common.RadioAccessTechnology - 98, // 86: org.lfedge.eve.info.ZCellularStatus.connected_at:type_name -> google.protobuf.Timestamp - 49, // 87: org.lfedge.eve.info.ZCellularStatus.bearers:type_name -> org.lfedge.eve.info.CellularBearer - 50, // 88: org.lfedge.eve.info.ZCellularStatus.profiles:type_name -> org.lfedge.eve.info.CellularProfile - 2, // 89: org.lfedge.eve.info.ZInfoDevSW.status:type_name -> org.lfedge.eve.info.ZSwState - 95, // 90: org.lfedge.eve.info.ZInfoDevSW.swErr:type_name -> org.lfedge.eve.info.ErrorInfo - 19, // 91: org.lfedge.eve.info.ZInfoDevSW.userStatus:type_name -> org.lfedge.eve.info.BaseOsStatus - 20, // 92: org.lfedge.eve.info.ZInfoDevSW.subStatus:type_name -> org.lfedge.eve.info.BaseOsSubStatus - 98, // 93: org.lfedge.eve.info.ZInfoSnapshot.create_time:type_name -> google.protobuf.Timestamp - 21, // 94: org.lfedge.eve.info.ZInfoSnapshot.type:type_name -> org.lfedge.eve.info.SnapshotType - 95, // 95: org.lfedge.eve.info.ZInfoSnapshot.snap_err:type_name -> org.lfedge.eve.info.ErrorInfo - 22, // 96: org.lfedge.eve.info.ZInfoClusterNode.node_status:type_name -> org.lfedge.eve.info.ZInfoClusterNodeStatus - 38, // 97: org.lfedge.eve.info.ZInfoApp.softwareList:type_name -> org.lfedge.eve.info.ZInfoSW - 98, // 98: org.lfedge.eve.info.ZInfoApp.bootTime:type_name -> google.protobuf.Timestamp - 31, // 99: org.lfedge.eve.info.ZInfoApp.assignedAdapters:type_name -> org.lfedge.eve.info.ZioBundle - 95, // 100: org.lfedge.eve.info.ZInfoApp.appErr:type_name -> org.lfedge.eve.info.ErrorInfo - 2, // 101: org.lfedge.eve.info.ZInfoApp.state:type_name -> org.lfedge.eve.info.ZSwState - 35, // 102: org.lfedge.eve.info.ZInfoApp.network:type_name -> org.lfedge.eve.info.ZInfoNetwork - 66, // 103: org.lfedge.eve.info.ZInfoApp.snapshots:type_name -> org.lfedge.eve.info.ZInfoSnapshot - 23, // 104: org.lfedge.eve.info.ZInfoVpnLink.state:type_name -> org.lfedge.eve.info.ZInfoVpnState - 69, // 105: org.lfedge.eve.info.ZInfoVpnLink.lInfo:type_name -> org.lfedge.eve.info.ZInfoVpnLinkInfo - 69, // 106: org.lfedge.eve.info.ZInfoVpnLink.rInfo:type_name -> org.lfedge.eve.info.ZInfoVpnLinkInfo - 23, // 107: org.lfedge.eve.info.ZInfoVpnConn.state:type_name -> org.lfedge.eve.info.ZInfoVpnState - 71, // 108: org.lfedge.eve.info.ZInfoVpnConn.lInfo:type_name -> org.lfedge.eve.info.ZInfoVpnEndPoint - 71, // 109: org.lfedge.eve.info.ZInfoVpnConn.rInfo:type_name -> org.lfedge.eve.info.ZInfoVpnEndPoint - 70, // 110: org.lfedge.eve.info.ZInfoVpnConn.links:type_name -> org.lfedge.eve.info.ZInfoVpnLink - 72, // 111: org.lfedge.eve.info.ZInfoVpn.conn:type_name -> org.lfedge.eve.info.ZInfoVpnConn - 98, // 112: org.lfedge.eve.info.ZInfoNetworkInstance.upTimeStamp:type_name -> google.protobuf.Timestamp - 38, // 113: org.lfedge.eve.info.ZInfoNetworkInstance.softwareList:type_name -> org.lfedge.eve.info.ZInfoSW - 29, // 114: org.lfedge.eve.info.ZInfoNetworkInstance.ipAssignments:type_name -> org.lfedge.eve.info.ZmetIPAssignmentEntry - 30, // 115: org.lfedge.eve.info.ZInfoNetworkInstance.vifs:type_name -> org.lfedge.eve.info.ZmetVifInfo - 31, // 116: org.lfedge.eve.info.ZInfoNetworkInstance.assignedAdapters:type_name -> org.lfedge.eve.info.ZioBundle - 73, // 117: org.lfedge.eve.info.ZInfoNetworkInstance.vinfo:type_name -> org.lfedge.eve.info.ZInfoVpn - 95, // 118: org.lfedge.eve.info.ZInfoNetworkInstance.networkErr:type_name -> org.lfedge.eve.info.ErrorInfo - 24, // 119: org.lfedge.eve.info.ZInfoNetworkInstance.state:type_name -> org.lfedge.eve.info.ZNetworkInstanceState - 75, // 120: org.lfedge.eve.info.ZInfoNetworkInstance.ip_routes:type_name -> org.lfedge.eve.info.IPRoute - 98, // 121: org.lfedge.eve.info.UsageInfo.createTime:type_name -> google.protobuf.Timestamp - 98, // 122: org.lfedge.eve.info.UsageInfo.lastRefcountChangeTime:type_name -> google.protobuf.Timestamp - 76, // 123: org.lfedge.eve.info.ZInfoVolume.usage:type_name -> org.lfedge.eve.info.UsageInfo - 77, // 124: org.lfedge.eve.info.ZInfoVolume.resources:type_name -> org.lfedge.eve.info.VolumeResources - 2, // 125: org.lfedge.eve.info.ZInfoVolume.state:type_name -> org.lfedge.eve.info.ZSwState - 95, // 126: org.lfedge.eve.info.ZInfoVolume.volumeErr:type_name -> org.lfedge.eve.info.ErrorInfo - 79, // 127: org.lfedge.eve.info.ZInfoContentTree.resources:type_name -> org.lfedge.eve.info.ContentResources - 76, // 128: org.lfedge.eve.info.ZInfoContentTree.usage:type_name -> org.lfedge.eve.info.UsageInfo - 2, // 129: org.lfedge.eve.info.ZInfoContentTree.state:type_name -> org.lfedge.eve.info.ZSwState - 95, // 130: org.lfedge.eve.info.ZInfoContentTree.err:type_name -> org.lfedge.eve.info.ErrorInfo - 79, // 131: org.lfedge.eve.info.ZInfoBlob.resources:type_name -> org.lfedge.eve.info.ContentResources - 76, // 132: org.lfedge.eve.info.ZInfoBlob.usage:type_name -> org.lfedge.eve.info.UsageInfo - 2, // 133: org.lfedge.eve.info.ZInfoBlob.state:type_name -> org.lfedge.eve.info.ZSwState - 95, // 134: org.lfedge.eve.info.ZInfoBlob.err:type_name -> org.lfedge.eve.info.ErrorInfo - 81, // 135: org.lfedge.eve.info.ZInfoBlobList.blob:type_name -> org.lfedge.eve.info.ZInfoBlob - 1, // 136: org.lfedge.eve.info.ZInfoMsg.ztype:type_name -> org.lfedge.eve.info.ZInfoTypes - 54, // 137: org.lfedge.eve.info.ZInfoMsg.dinfo:type_name -> org.lfedge.eve.info.ZInfoDevice - 68, // 138: org.lfedge.eve.info.ZInfoMsg.ainfo:type_name -> org.lfedge.eve.info.ZInfoApp - 74, // 139: org.lfedge.eve.info.ZInfoMsg.niinfo:type_name -> org.lfedge.eve.info.ZInfoNetworkInstance - 78, // 140: org.lfedge.eve.info.ZInfoMsg.vinfo:type_name -> org.lfedge.eve.info.ZInfoVolume - 80, // 141: org.lfedge.eve.info.ZInfoMsg.cinfo:type_name -> org.lfedge.eve.info.ZInfoContentTree - 82, // 142: org.lfedge.eve.info.ZInfoMsg.binfo:type_name -> org.lfedge.eve.info.ZInfoBlobList - 85, // 143: org.lfedge.eve.info.ZInfoMsg.amdinfo:type_name -> org.lfedge.eve.info.ZInfoAppInstMetaData - 86, // 144: org.lfedge.eve.info.ZInfoMsg.evinfo:type_name -> org.lfedge.eve.info.ZInfoEdgeview - 90, // 145: org.lfedge.eve.info.ZInfoMsg.hwinfo:type_name -> org.lfedge.eve.info.ZInfoHardware - 87, // 146: org.lfedge.eve.info.ZInfoMsg.locinfo:type_name -> org.lfedge.eve.info.ZInfoLocation - 105, // 147: org.lfedge.eve.info.ZInfoMsg.patchInfo:type_name -> org.lfedge.eve.info.ZInfoPatchEnvelope - 67, // 148: org.lfedge.eve.info.ZInfoMsg.cluster_node:type_name -> org.lfedge.eve.info.ZInfoClusterNode - 106, // 149: org.lfedge.eve.info.ZInfoMsg.ntp_sources:type_name -> org.lfedge.eve.info.ZInfoNTPSources - 89, // 150: org.lfedge.eve.info.ZInfoMsg.cluster_info:type_name -> org.lfedge.eve.info.ZInfoKubeCluster - 88, // 151: org.lfedge.eve.info.ZInfoMsg.cluster_update_info:type_name -> org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus - 98, // 152: org.lfedge.eve.info.ZInfoMsg.atTimeStamp:type_name -> google.protobuf.Timestamp - 17, // 153: org.lfedge.eve.info.ZInfoAppInstMetaData.type:type_name -> org.lfedge.eve.info.AppInstMetaDataType - 98, // 154: org.lfedge.eve.info.ZInfoEdgeview.expire_time:type_name -> google.protobuf.Timestamp - 98, // 155: org.lfedge.eve.info.ZInfoEdgeview.started_time:type_name -> google.protobuf.Timestamp - 98, // 156: org.lfedge.eve.info.ZInfoLocation.utc_timestamp:type_name -> google.protobuf.Timestamp - 25, // 157: org.lfedge.eve.info.ZInfoLocation.horizontal_reliability:type_name -> org.lfedge.eve.info.LocReliability - 25, // 158: org.lfedge.eve.info.ZInfoLocation.vertical_reliability:type_name -> org.lfedge.eve.info.LocReliability - 107, // 159: org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus.component:type_name -> org.lfedge.eve.info.KubeComp - 108, // 160: org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus.status:type_name -> org.lfedge.eve.info.KubeCompUpdateStatus - 95, // 161: org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus.error:type_name -> org.lfedge.eve.info.ErrorInfo - 109, // 162: org.lfedge.eve.info.ZInfoKubeCluster.nodes:type_name -> org.lfedge.eve.info.KubeNodeInfo - 110, // 163: org.lfedge.eve.info.ZInfoKubeCluster.pod_name_spaces:type_name -> org.lfedge.eve.info.KubePodNameSpaceInfo - 111, // 164: org.lfedge.eve.info.ZInfoKubeCluster.eve_apps:type_name -> org.lfedge.eve.info.KubeEVEAppPodInfo - 112, // 165: org.lfedge.eve.info.ZInfoKubeCluster.storage:type_name -> org.lfedge.eve.info.KubeStorageInfo - 113, // 166: org.lfedge.eve.info.ZInfoKubeCluster.eve_vm_apps:type_name -> org.lfedge.eve.info.KubeVMIInfo - 27, // 167: org.lfedge.eve.info.ZInfoHardware.disks:type_name -> org.lfedge.eve.info.StorageDiskInfo - 114, // 168: org.lfedge.eve.info.ZInfoHardware.inventory:type_name -> org.lfedge.eve.info.HardwareInventory - 42, // 169: org.lfedge.eve.info.ZInfoConfigItemStatus.ConfigItemsEntry.value:type_name -> org.lfedge.eve.info.ZInfoConfigItem - 42, // 170: org.lfedge.eve.info.ZInfoConfigItemStatus.UnknownConfigItemsEntry.value:type_name -> org.lfedge.eve.info.ZInfoConfigItem - 171, // [171:171] is the sub-list for method output_type - 171, // [171:171] is the sub-list for method input_type - 171, // [171:171] is the sub-list for extension type_name - 171, // [171:171] is the sub-list for extension extendee - 0, // [0:171] is the sub-list for field type_name + 52, // 35: org.lfedge.eve.info.StorageInfo.disks:type_name -> org.lfedge.eve.info.StorageDiskState + 53, // 36: org.lfedge.eve.info.StorageInfo.children:type_name -> org.lfedge.eve.info.StorageChildren + 13, // 37: org.lfedge.eve.info.CPUPoolUtilization.kind:type_name -> org.lfedge.eve.info.CPUPoolKind + 35, // 38: org.lfedge.eve.info.ZInfoDevice.minfo:type_name -> org.lfedge.eve.info.ZInfoManufacturer + 36, // 39: org.lfedge.eve.info.ZInfoDevice.network:type_name -> org.lfedge.eve.info.ZInfoNetwork + 32, // 40: org.lfedge.eve.info.ZInfoDevice.assignableAdapters:type_name -> org.lfedge.eve.info.ZioBundle + 38, // 41: org.lfedge.eve.info.ZInfoDevice.dns:type_name -> org.lfedge.eve.info.ZInfoDNS + 67, // 42: org.lfedge.eve.info.ZInfoDevice.storageList:type_name -> org.lfedge.eve.info.ZInfoStorage + 100, // 43: org.lfedge.eve.info.ZInfoDevice.bootTime:type_name -> google.protobuf.Timestamp + 66, // 44: org.lfedge.eve.info.ZInfoDevice.swList:type_name -> org.lfedge.eve.info.ZInfoDevSW + 29, // 45: org.lfedge.eve.info.ZInfoDevice.metricItems:type_name -> org.lfedge.eve.info.deprecatedMetricItem + 100, // 46: org.lfedge.eve.info.ZInfoDevice.lastRebootTime:type_name -> google.protobuf.Timestamp + 59, // 47: org.lfedge.eve.info.ZInfoDevice.systemAdapter:type_name -> org.lfedge.eve.info.SystemAdapterInfo + 3, // 48: org.lfedge.eve.info.ZInfoDevice.HSMStatus:type_name -> org.lfedge.eve.info.HwSecurityModuleStatus + 41, // 49: org.lfedge.eve.info.ZInfoDevice.dataSecAtRestInfo:type_name -> org.lfedge.eve.info.DataSecAtRest + 42, // 50: org.lfedge.eve.info.ZInfoDevice.sec_info:type_name -> org.lfedge.eve.info.SecurityInfo + 44, // 51: org.lfedge.eve.info.ZInfoDevice.configItemStatus:type_name -> org.lfedge.eve.info.ZInfoConfigItemStatus + 45, // 52: org.lfedge.eve.info.ZInfoDevice.appInstances:type_name -> org.lfedge.eve.info.ZInfoAppInstance + 15, // 53: org.lfedge.eve.info.ZInfoDevice.last_boot_reason:type_name -> org.lfedge.eve.info.BootReason + 48, // 54: org.lfedge.eve.info.ZInfoDevice.cell_radios:type_name -> org.lfedge.eve.info.ZCellularModuleInfo + 47, // 55: org.lfedge.eve.info.ZInfoDevice.sims:type_name -> org.lfedge.eve.info.ZSimcardInfo + 46, // 56: org.lfedge.eve.info.ZInfoDevice.tasks:type_name -> org.lfedge.eve.info.ZInfoDeviceTasks + 16, // 57: org.lfedge.eve.info.ZInfoDevice.maintenance_mode_reason:type_name -> org.lfedge.eve.info.MaintenanceModeReason + 86, // 58: org.lfedge.eve.info.ZInfoDevice.capabilities:type_name -> org.lfedge.eve.info.Capabilities + 9, // 59: org.lfedge.eve.info.ZInfoDevice.state:type_name -> org.lfedge.eve.info.ZDeviceState + 16, // 60: org.lfedge.eve.info.ZInfoDevice.maintenance_mode_reasons:type_name -> org.lfedge.eve.info.MaintenanceModeReason + 54, // 61: org.lfedge.eve.info.ZInfoDevice.storage_info:type_name -> org.lfedge.eve.info.StorageInfo + 58, // 62: org.lfedge.eve.info.ZInfoDevice.attestation_info:type_name -> org.lfedge.eve.info.AttestationInfo + 14, // 63: org.lfedge.eve.info.ZInfoDevice.api_capability:type_name -> org.lfedge.eve.info.APICapability + 57, // 64: org.lfedge.eve.info.ZInfoDevice.optional_capabilities:type_name -> org.lfedge.eve.info.OptionalCapabilities + 102, // 65: org.lfedge.eve.info.ZInfoDevice.enrolled_certs:type_name -> org.lfedge.eve.info.CertInfo + 55, // 66: org.lfedge.eve.info.ZInfoDevice.cpu_pools:type_name -> org.lfedge.eve.info.CPUPoolUtilization + 17, // 67: org.lfedge.eve.info.AttestationInfo.state:type_name -> org.lfedge.eve.info.AttestationState + 97, // 68: org.lfedge.eve.info.AttestationInfo.error:type_name -> org.lfedge.eve.info.ErrorInfo + 60, // 69: org.lfedge.eve.info.SystemAdapterInfo.status:type_name -> org.lfedge.eve.info.DevicePortStatus + 100, // 70: org.lfedge.eve.info.DevicePortStatus.timePriority:type_name -> google.protobuf.Timestamp + 100, // 71: org.lfedge.eve.info.DevicePortStatus.lastFailed:type_name -> google.protobuf.Timestamp + 100, // 72: org.lfedge.eve.info.DevicePortStatus.lastSucceeded:type_name -> google.protobuf.Timestamp + 61, // 73: org.lfedge.eve.info.DevicePortStatus.ports:type_name -> org.lfedge.eve.info.DevicePort + 62, // 74: org.lfedge.eve.info.DevicePort.proxy:type_name -> org.lfedge.eve.info.ProxyStatus + 38, // 75: org.lfedge.eve.info.DevicePort.dns:type_name -> org.lfedge.eve.info.ZInfoDNS + 37, // 76: org.lfedge.eve.info.DevicePort.location:type_name -> org.lfedge.eve.info.GeoLoc + 97, // 77: org.lfedge.eve.info.DevicePort.err:type_name -> org.lfedge.eve.info.ErrorInfo + 96, // 78: org.lfedge.eve.info.DevicePort.usage:type_name -> org.lfedge.eve.common.PhyIoMemberUsage + 64, // 79: org.lfedge.eve.info.DevicePort.wireless_status:type_name -> org.lfedge.eve.info.WirelessStatus + 103, // 80: org.lfedge.eve.info.DevicePort.config_source:type_name -> org.lfedge.eve.common.PortConfigSource + 104, // 81: org.lfedge.eve.info.DevicePort.pnac_status:type_name -> org.lfedge.eve.info.PNACStatus + 105, // 82: org.lfedge.eve.info.DevicePort.bond_status:type_name -> org.lfedge.eve.info.BondStatus + 63, // 83: org.lfedge.eve.info.ProxyStatus.proxies:type_name -> org.lfedge.eve.info.ProxyEntry + 19, // 84: org.lfedge.eve.info.WirelessStatus.type:type_name -> org.lfedge.eve.info.WirelessType + 65, // 85: org.lfedge.eve.info.WirelessStatus.cellular:type_name -> org.lfedge.eve.info.ZCellularStatus + 49, // 86: org.lfedge.eve.info.ZCellularStatus.providers:type_name -> org.lfedge.eve.info.ZCellularProvider + 106, // 87: org.lfedge.eve.info.ZCellularStatus.current_rats:type_name -> org.lfedge.eve.common.RadioAccessTechnology + 100, // 88: org.lfedge.eve.info.ZCellularStatus.connected_at:type_name -> google.protobuf.Timestamp + 50, // 89: org.lfedge.eve.info.ZCellularStatus.bearers:type_name -> org.lfedge.eve.info.CellularBearer + 51, // 90: org.lfedge.eve.info.ZCellularStatus.profiles:type_name -> org.lfedge.eve.info.CellularProfile + 2, // 91: org.lfedge.eve.info.ZInfoDevSW.status:type_name -> org.lfedge.eve.info.ZSwState + 97, // 92: org.lfedge.eve.info.ZInfoDevSW.swErr:type_name -> org.lfedge.eve.info.ErrorInfo + 20, // 93: org.lfedge.eve.info.ZInfoDevSW.userStatus:type_name -> org.lfedge.eve.info.BaseOsStatus + 21, // 94: org.lfedge.eve.info.ZInfoDevSW.subStatus:type_name -> org.lfedge.eve.info.BaseOsSubStatus + 100, // 95: org.lfedge.eve.info.ZInfoSnapshot.create_time:type_name -> google.protobuf.Timestamp + 22, // 96: org.lfedge.eve.info.ZInfoSnapshot.type:type_name -> org.lfedge.eve.info.SnapshotType + 97, // 97: org.lfedge.eve.info.ZInfoSnapshot.snap_err:type_name -> org.lfedge.eve.info.ErrorInfo + 23, // 98: org.lfedge.eve.info.ZInfoClusterNode.node_status:type_name -> org.lfedge.eve.info.ZInfoClusterNodeStatus + 39, // 99: org.lfedge.eve.info.ZInfoApp.softwareList:type_name -> org.lfedge.eve.info.ZInfoSW + 100, // 100: org.lfedge.eve.info.ZInfoApp.bootTime:type_name -> google.protobuf.Timestamp + 32, // 101: org.lfedge.eve.info.ZInfoApp.assignedAdapters:type_name -> org.lfedge.eve.info.ZioBundle + 97, // 102: org.lfedge.eve.info.ZInfoApp.appErr:type_name -> org.lfedge.eve.info.ErrorInfo + 2, // 103: org.lfedge.eve.info.ZInfoApp.state:type_name -> org.lfedge.eve.info.ZSwState + 36, // 104: org.lfedge.eve.info.ZInfoApp.network:type_name -> org.lfedge.eve.info.ZInfoNetwork + 68, // 105: org.lfedge.eve.info.ZInfoApp.snapshots:type_name -> org.lfedge.eve.info.ZInfoSnapshot + 24, // 106: org.lfedge.eve.info.ZInfoVpnLink.state:type_name -> org.lfedge.eve.info.ZInfoVpnState + 71, // 107: org.lfedge.eve.info.ZInfoVpnLink.lInfo:type_name -> org.lfedge.eve.info.ZInfoVpnLinkInfo + 71, // 108: org.lfedge.eve.info.ZInfoVpnLink.rInfo:type_name -> org.lfedge.eve.info.ZInfoVpnLinkInfo + 24, // 109: org.lfedge.eve.info.ZInfoVpnConn.state:type_name -> org.lfedge.eve.info.ZInfoVpnState + 73, // 110: org.lfedge.eve.info.ZInfoVpnConn.lInfo:type_name -> org.lfedge.eve.info.ZInfoVpnEndPoint + 73, // 111: org.lfedge.eve.info.ZInfoVpnConn.rInfo:type_name -> org.lfedge.eve.info.ZInfoVpnEndPoint + 72, // 112: org.lfedge.eve.info.ZInfoVpnConn.links:type_name -> org.lfedge.eve.info.ZInfoVpnLink + 74, // 113: org.lfedge.eve.info.ZInfoVpn.conn:type_name -> org.lfedge.eve.info.ZInfoVpnConn + 100, // 114: org.lfedge.eve.info.ZInfoNetworkInstance.upTimeStamp:type_name -> google.protobuf.Timestamp + 39, // 115: org.lfedge.eve.info.ZInfoNetworkInstance.softwareList:type_name -> org.lfedge.eve.info.ZInfoSW + 30, // 116: org.lfedge.eve.info.ZInfoNetworkInstance.ipAssignments:type_name -> org.lfedge.eve.info.ZmetIPAssignmentEntry + 31, // 117: org.lfedge.eve.info.ZInfoNetworkInstance.vifs:type_name -> org.lfedge.eve.info.ZmetVifInfo + 32, // 118: org.lfedge.eve.info.ZInfoNetworkInstance.assignedAdapters:type_name -> org.lfedge.eve.info.ZioBundle + 75, // 119: org.lfedge.eve.info.ZInfoNetworkInstance.vinfo:type_name -> org.lfedge.eve.info.ZInfoVpn + 97, // 120: org.lfedge.eve.info.ZInfoNetworkInstance.networkErr:type_name -> org.lfedge.eve.info.ErrorInfo + 25, // 121: org.lfedge.eve.info.ZInfoNetworkInstance.state:type_name -> org.lfedge.eve.info.ZNetworkInstanceState + 77, // 122: org.lfedge.eve.info.ZInfoNetworkInstance.ip_routes:type_name -> org.lfedge.eve.info.IPRoute + 100, // 123: org.lfedge.eve.info.UsageInfo.createTime:type_name -> google.protobuf.Timestamp + 100, // 124: org.lfedge.eve.info.UsageInfo.lastRefcountChangeTime:type_name -> google.protobuf.Timestamp + 78, // 125: org.lfedge.eve.info.ZInfoVolume.usage:type_name -> org.lfedge.eve.info.UsageInfo + 79, // 126: org.lfedge.eve.info.ZInfoVolume.resources:type_name -> org.lfedge.eve.info.VolumeResources + 2, // 127: org.lfedge.eve.info.ZInfoVolume.state:type_name -> org.lfedge.eve.info.ZSwState + 97, // 128: org.lfedge.eve.info.ZInfoVolume.volumeErr:type_name -> org.lfedge.eve.info.ErrorInfo + 81, // 129: org.lfedge.eve.info.ZInfoContentTree.resources:type_name -> org.lfedge.eve.info.ContentResources + 78, // 130: org.lfedge.eve.info.ZInfoContentTree.usage:type_name -> org.lfedge.eve.info.UsageInfo + 2, // 131: org.lfedge.eve.info.ZInfoContentTree.state:type_name -> org.lfedge.eve.info.ZSwState + 97, // 132: org.lfedge.eve.info.ZInfoContentTree.err:type_name -> org.lfedge.eve.info.ErrorInfo + 81, // 133: org.lfedge.eve.info.ZInfoBlob.resources:type_name -> org.lfedge.eve.info.ContentResources + 78, // 134: org.lfedge.eve.info.ZInfoBlob.usage:type_name -> org.lfedge.eve.info.UsageInfo + 2, // 135: org.lfedge.eve.info.ZInfoBlob.state:type_name -> org.lfedge.eve.info.ZSwState + 97, // 136: org.lfedge.eve.info.ZInfoBlob.err:type_name -> org.lfedge.eve.info.ErrorInfo + 83, // 137: org.lfedge.eve.info.ZInfoBlobList.blob:type_name -> org.lfedge.eve.info.ZInfoBlob + 1, // 138: org.lfedge.eve.info.ZInfoMsg.ztype:type_name -> org.lfedge.eve.info.ZInfoTypes + 56, // 139: org.lfedge.eve.info.ZInfoMsg.dinfo:type_name -> org.lfedge.eve.info.ZInfoDevice + 70, // 140: org.lfedge.eve.info.ZInfoMsg.ainfo:type_name -> org.lfedge.eve.info.ZInfoApp + 76, // 141: org.lfedge.eve.info.ZInfoMsg.niinfo:type_name -> org.lfedge.eve.info.ZInfoNetworkInstance + 80, // 142: org.lfedge.eve.info.ZInfoMsg.vinfo:type_name -> org.lfedge.eve.info.ZInfoVolume + 82, // 143: org.lfedge.eve.info.ZInfoMsg.cinfo:type_name -> org.lfedge.eve.info.ZInfoContentTree + 84, // 144: org.lfedge.eve.info.ZInfoMsg.binfo:type_name -> org.lfedge.eve.info.ZInfoBlobList + 87, // 145: org.lfedge.eve.info.ZInfoMsg.amdinfo:type_name -> org.lfedge.eve.info.ZInfoAppInstMetaData + 88, // 146: org.lfedge.eve.info.ZInfoMsg.evinfo:type_name -> org.lfedge.eve.info.ZInfoEdgeview + 92, // 147: org.lfedge.eve.info.ZInfoMsg.hwinfo:type_name -> org.lfedge.eve.info.ZInfoHardware + 89, // 148: org.lfedge.eve.info.ZInfoMsg.locinfo:type_name -> org.lfedge.eve.info.ZInfoLocation + 107, // 149: org.lfedge.eve.info.ZInfoMsg.patchInfo:type_name -> org.lfedge.eve.info.ZInfoPatchEnvelope + 69, // 150: org.lfedge.eve.info.ZInfoMsg.cluster_node:type_name -> org.lfedge.eve.info.ZInfoClusterNode + 108, // 151: org.lfedge.eve.info.ZInfoMsg.ntp_sources:type_name -> org.lfedge.eve.info.ZInfoNTPSources + 91, // 152: org.lfedge.eve.info.ZInfoMsg.cluster_info:type_name -> org.lfedge.eve.info.ZInfoKubeCluster + 90, // 153: org.lfedge.eve.info.ZInfoMsg.cluster_update_info:type_name -> org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus + 100, // 154: org.lfedge.eve.info.ZInfoMsg.atTimeStamp:type_name -> google.protobuf.Timestamp + 18, // 155: org.lfedge.eve.info.ZInfoAppInstMetaData.type:type_name -> org.lfedge.eve.info.AppInstMetaDataType + 100, // 156: org.lfedge.eve.info.ZInfoEdgeview.expire_time:type_name -> google.protobuf.Timestamp + 100, // 157: org.lfedge.eve.info.ZInfoEdgeview.started_time:type_name -> google.protobuf.Timestamp + 100, // 158: org.lfedge.eve.info.ZInfoLocation.utc_timestamp:type_name -> google.protobuf.Timestamp + 26, // 159: org.lfedge.eve.info.ZInfoLocation.horizontal_reliability:type_name -> org.lfedge.eve.info.LocReliability + 26, // 160: org.lfedge.eve.info.ZInfoLocation.vertical_reliability:type_name -> org.lfedge.eve.info.LocReliability + 109, // 161: org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus.component:type_name -> org.lfedge.eve.info.KubeComp + 110, // 162: org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus.status:type_name -> org.lfedge.eve.info.KubeCompUpdateStatus + 97, // 163: org.lfedge.eve.info.ZInfoKubeClusterUpdateStatus.error:type_name -> org.lfedge.eve.info.ErrorInfo + 111, // 164: org.lfedge.eve.info.ZInfoKubeCluster.nodes:type_name -> org.lfedge.eve.info.KubeNodeInfo + 112, // 165: org.lfedge.eve.info.ZInfoKubeCluster.pod_name_spaces:type_name -> org.lfedge.eve.info.KubePodNameSpaceInfo + 113, // 166: org.lfedge.eve.info.ZInfoKubeCluster.eve_apps:type_name -> org.lfedge.eve.info.KubeEVEAppPodInfo + 114, // 167: org.lfedge.eve.info.ZInfoKubeCluster.storage:type_name -> org.lfedge.eve.info.KubeStorageInfo + 115, // 168: org.lfedge.eve.info.ZInfoKubeCluster.eve_vm_apps:type_name -> org.lfedge.eve.info.KubeVMIInfo + 28, // 169: org.lfedge.eve.info.ZInfoHardware.disks:type_name -> org.lfedge.eve.info.StorageDiskInfo + 116, // 170: org.lfedge.eve.info.ZInfoHardware.inventory:type_name -> org.lfedge.eve.info.HardwareInventory + 43, // 171: org.lfedge.eve.info.ZInfoConfigItemStatus.ConfigItemsEntry.value:type_name -> org.lfedge.eve.info.ZInfoConfigItem + 43, // 172: org.lfedge.eve.info.ZInfoConfigItemStatus.UnknownConfigItemsEntry.value:type_name -> org.lfedge.eve.info.ZInfoConfigItem + 173, // [173:173] is the sub-list for method output_type + 173, // [173:173] is the sub-list for method input_type + 173, // [173:173] is the sub-list for extension type_name + 173, // [173:173] is the sub-list for extension extendee + 0, // [0:173] is the sub-list for field type_name } func init() { file_info_info_proto_init() } @@ -10706,7 +10961,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoDevice); i { + switch v := v.(*CPUPoolUtilization); i { case 0: return &v.state case 1: @@ -10718,7 +10973,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OptionalCapabilities); i { + switch v := v.(*ZInfoDevice); i { case 0: return &v.state case 1: @@ -10730,7 +10985,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AttestationInfo); i { + switch v := v.(*OptionalCapabilities); i { case 0: return &v.state case 1: @@ -10742,7 +10997,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SystemAdapterInfo); i { + switch v := v.(*AttestationInfo); i { case 0: return &v.state case 1: @@ -10754,7 +11009,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DevicePortStatus); i { + switch v := v.(*SystemAdapterInfo); i { case 0: return &v.state case 1: @@ -10766,7 +11021,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DevicePort); i { + switch v := v.(*DevicePortStatus); i { case 0: return &v.state case 1: @@ -10778,7 +11033,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProxyStatus); i { + switch v := v.(*DevicePort); i { case 0: return &v.state case 1: @@ -10790,7 +11045,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProxyEntry); i { + switch v := v.(*ProxyStatus); i { case 0: return &v.state case 1: @@ -10802,7 +11057,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WirelessStatus); i { + switch v := v.(*ProxyEntry); i { case 0: return &v.state case 1: @@ -10814,7 +11069,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZCellularStatus); i { + switch v := v.(*WirelessStatus); i { case 0: return &v.state case 1: @@ -10826,7 +11081,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoDevSW); i { + switch v := v.(*ZCellularStatus); i { case 0: return &v.state case 1: @@ -10838,7 +11093,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoStorage); i { + switch v := v.(*ZInfoDevSW); i { case 0: return &v.state case 1: @@ -10850,7 +11105,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoSnapshot); i { + switch v := v.(*ZInfoStorage); i { case 0: return &v.state case 1: @@ -10862,7 +11117,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoClusterNode); i { + switch v := v.(*ZInfoSnapshot); i { case 0: return &v.state case 1: @@ -10874,7 +11129,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoApp); i { + switch v := v.(*ZInfoClusterNode); i { case 0: return &v.state case 1: @@ -10886,7 +11141,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoVpnLinkInfo); i { + switch v := v.(*ZInfoApp); i { case 0: return &v.state case 1: @@ -10898,7 +11153,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoVpnLink); i { + switch v := v.(*ZInfoVpnLinkInfo); i { case 0: return &v.state case 1: @@ -10910,7 +11165,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoVpnEndPoint); i { + switch v := v.(*ZInfoVpnLink); i { case 0: return &v.state case 1: @@ -10922,7 +11177,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoVpnConn); i { + switch v := v.(*ZInfoVpnEndPoint); i { case 0: return &v.state case 1: @@ -10934,7 +11189,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoVpn); i { + switch v := v.(*ZInfoVpnConn); i { case 0: return &v.state case 1: @@ -10946,7 +11201,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoNetworkInstance); i { + switch v := v.(*ZInfoVpn); i { case 0: return &v.state case 1: @@ -10958,7 +11213,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*IPRoute); i { + switch v := v.(*ZInfoNetworkInstance); i { case 0: return &v.state case 1: @@ -10970,7 +11225,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UsageInfo); i { + switch v := v.(*IPRoute); i { case 0: return &v.state case 1: @@ -10982,7 +11237,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VolumeResources); i { + switch v := v.(*UsageInfo); i { case 0: return &v.state case 1: @@ -10994,7 +11249,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoVolume); i { + switch v := v.(*VolumeResources); i { case 0: return &v.state case 1: @@ -11006,7 +11261,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ContentResources); i { + switch v := v.(*ZInfoVolume); i { case 0: return &v.state case 1: @@ -11018,7 +11273,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoContentTree); i { + switch v := v.(*ContentResources); i { case 0: return &v.state case 1: @@ -11030,7 +11285,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoBlob); i { + switch v := v.(*ZInfoContentTree); i { case 0: return &v.state case 1: @@ -11042,7 +11297,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoBlobList); i { + switch v := v.(*ZInfoBlob); i { case 0: return &v.state case 1: @@ -11054,7 +11309,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoMsg); i { + switch v := v.(*ZInfoBlobList); i { case 0: return &v.state case 1: @@ -11066,7 +11321,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Capabilities); i { + switch v := v.(*ZInfoMsg); i { case 0: return &v.state case 1: @@ -11078,7 +11333,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoAppInstMetaData); i { + switch v := v.(*Capabilities); i { case 0: return &v.state case 1: @@ -11090,7 +11345,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoEdgeview); i { + switch v := v.(*ZInfoAppInstMetaData); i { case 0: return &v.state case 1: @@ -11102,7 +11357,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoLocation); i { + switch v := v.(*ZInfoEdgeview); i { case 0: return &v.state case 1: @@ -11114,7 +11369,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoKubeClusterUpdateStatus); i { + switch v := v.(*ZInfoLocation); i { case 0: return &v.state case 1: @@ -11126,7 +11381,7 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ZInfoKubeCluster); i { + switch v := v.(*ZInfoKubeClusterUpdateStatus); i { case 0: return &v.state case 1: @@ -11138,6 +11393,18 @@ func file_info_info_proto_init() { } } file_info_info_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ZInfoKubeCluster); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_info_info_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ZInfoHardware); i { case 0: return &v.state @@ -11157,10 +11424,10 @@ func file_info_info_proto_init() { (*DeprecatedMetricItem_FloatValue)(nil), (*DeprecatedMetricItem_StringValue)(nil), } - file_info_info_proto_msgTypes[48].OneofWrappers = []interface{}{ + file_info_info_proto_msgTypes[49].OneofWrappers = []interface{}{ (*ZInfoNetworkInstance_Vinfo)(nil), } - file_info_info_proto_msgTypes[57].OneofWrappers = []interface{}{ + file_info_info_proto_msgTypes[58].OneofWrappers = []interface{}{ (*ZInfoMsg_Dinfo)(nil), (*ZInfoMsg_Ainfo)(nil), (*ZInfoMsg_Niinfo)(nil), @@ -11182,8 +11449,8 @@ func file_info_info_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_info_info_proto_rawDesc, - NumEnums: 26, - NumMessages: 67, + NumEnums: 27, + NumMessages: 68, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/pillar/vendor/modules.txt b/pkg/pillar/vendor/modules.txt index 8d25b2aeb78..f48432c358a 100644 --- a/pkg/pillar/vendor/modules.txt +++ b/pkg/pillar/vendor/modules.txt @@ -866,7 +866,7 @@ github.com/leodido/go-urn github.com/lf-edge/edge-containers/pkg/registry github.com/lf-edge/edge-containers/pkg/resolver github.com/lf-edge/edge-containers/pkg/tgz -# github.com/lf-edge/eve-api/go v0.0.0-20260812180240-99d02ddcfcb0 +# github.com/lf-edge/eve-api/go v0.0.0-20260812180240-99d02ddcfcb0 => github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc ## explicit; go 1.21.1 github.com/lf-edge/eve-api/go/attest github.com/lf-edge/eve-api/go/auth @@ -2424,3 +2424,4 @@ tags.cncf.io/container-device-interface/specs-go # k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.33.5 # k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.33.5 # k8s.io/sample-controller => k8s.io/sample-controller v0.33.5 +# github.com/lf-edge/eve-api/go => github.com/rucoder/eve-api/go v0.0.0-20260817131207-e83592bee6cc