diff --git a/build/integration-in-docker-crio.sh b/build/integration-in-docker-crio.sh index 980134f5ce..47d686cc5b 100755 --- a/build/integration-in-docker-crio.sh +++ b/build/integration-in-docker-crio.sh @@ -76,7 +76,7 @@ function run_tests() { --cgroupns=host \ --pid=host \ --entrypoint="" \ - gcr.io/k8s-staging-test-infra/bootstrap:v20251209-855adc2699 \ + gcr.io/k8s-staging-test-infra/bootstrap@sha256:2e537f9aea810021ecd022bf085e99099a25fc4bb95de96eeb510e92a524b471 \ bash -c "export DEBIAN_FRONTEND=noninteractive && \ apt-get update && \ apt-get install -y $PACKAGES curl conntrack iptables dbus && \ diff --git a/cmd/go.mod b/cmd/go.mod index 19babfafbc..c3554c9e8f 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -6,7 +6,7 @@ go 1.24.0 // The github.com/google/cadvisor/cmd module is built using the Makefile // from a clone of the github.com/google/cadvisor repository, so we // always use the relative local source rather than specifying a module version. -require github.com/google/cadvisor v0.0.0 +require github.com/google/cadvisor v0.57.0 // Use the relative local source of the github.com/google/cadvisor library to build replace github.com/google/cadvisor => ../ @@ -107,9 +107,9 @@ require ( github.com/stretchr/objx v0.5.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.39.0 // indirect - go.opentelemetry.io/otel/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect golang.org/x/crypto v0.46.0 // indirect golang.org/x/net v0.48.0 // indirect golang.org/x/sys v0.39.0 // indirect diff --git a/cmd/go.sum b/cmd/go.sum index aa10e9d81f..49b8e5cf6d 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -260,16 +260,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= diff --git a/container/common/helpers.go b/container/common/helpers.go index 3b3d27f300..69e0adff2b 100644 --- a/container/common/helpers.go +++ b/container/common/helpers.go @@ -437,6 +437,10 @@ func (m deviceIdentifierMap) Find(major, minor uint64, namer DeviceNamer) string return s } s, _ := namer.DeviceName(major, minor) + if s == "" { + s = fmt.Sprintf("%d:%d", major, minor) + } + m[d] = s return s } diff --git a/container/common/helpers_test.go b/container/common/helpers_test.go index db9e13829f..8bb82f3cf6 100644 --- a/container/common/helpers_test.go +++ b/container/common/helpers_test.go @@ -250,6 +250,67 @@ func TestRemoveNetMetrics(t *testing.T) { } } +type deviceNamerFunc func(major, minor uint64) (string, bool) + +func (f deviceNamerFunc) DeviceName(major, minor uint64) (string, bool) { + return f(major, minor) +} + +func TestDeviceIdentifierMapFindFallsBackToMajorMinor(t *testing.T) { + for _, tc := range []struct { + name string + namer DeviceNamer + expected string + }{ + { + name: "unrecognized device", + namer: deviceNamerFunc(func(_, _ uint64) (string, bool) { + return "", false + }), + expected: "8:0", + }, + { + name: "recognized empty device", + namer: deviceNamerFunc(func(_, _ uint64) (string, bool) { + return "", true + }), + expected: "8:0", + }, + { + name: "recognized device", + namer: deviceNamerFunc(func(_, _ uint64) (string, bool) { + return "/dev/sda", true + }), + expected: "/dev/sda", + }, + } { + t.Run(tc.name, func(t *testing.T) { + devices := make(deviceIdentifierMap) + assert.Equal(t, tc.expected, devices.Find(8, 0, tc.namer)) + }) + } +} + +func TestAssignDeviceNamesToDiskStatsKeepsUnresolvedDevicesUnique(t *testing.T) { + stats := &info.DiskIoStats{ + IoServiceBytes: []info.PerDiskStats{ + {Major: 8, Minor: 0}, + {Major: 259, Minor: 1}, + }, + IoServiced: []info.PerDiskStats{ + {Major: 8, Minor: 0}, + }, + } + + AssignDeviceNamesToDiskStats(deviceNamerFunc(func(_, _ uint64) (string, bool) { + return "", false + }), stats) + + assert.Equal(t, "8:0", stats.IoServiceBytes[0].Device) + assert.Equal(t, "259:1", stats.IoServiceBytes[1].Device) + assert.Equal(t, "8:0", stats.IoServiced[0].Device) +} + func BenchmarkGetSpecCgroupV2(b *testing.B) { root, err := os.Getwd() if err != nil { diff --git a/container/raw/handler.go b/container/raw/handler.go index a53c018806..bf27c54f32 100644 --- a/container/raw/handler.go +++ b/container/raw/handler.go @@ -173,6 +173,8 @@ func fsToFsStats(fs *fs.Fs) info.FsStats { Inodes: inodes, InodesFree: inodesFree, Available: fs.Available, + Mountpoint: fs.Mountpoint, + AllMountpoints: fs.AllMountpoints, ReadsCompleted: fs.DiskStats.ReadsCompleted, ReadsMerged: fs.DiskStats.ReadsMerged, SectorsRead: fs.DiskStats.SectorsRead, @@ -215,9 +217,29 @@ func (h *rawContainerHandler) getFsStats(stats *info.ContainerStats) error { } if h.includedMetrics.Has(container.DiskUsageMetrics) { + // Build a map from host-side mount directory to container-side path + // from the external mounts (container hints) configuration. + hostToContainerPath := make(map[string]string, len(h.externalMounts)) + for _, m := range h.externalMounts { + if m.HostDir != "" && m.ContainerDir != "" { + hostToContainerPath[m.HostDir] = m.ContainerDir + } + } + for i := range filesystems { - fs := filesystems[i] - stats.Filesystem = append(stats.Filesystem, fsToFsStats(&fs)) + fsCopy := filesystems[i] + fsstat := fsToFsStats(&fsCopy) + if containerPath, ok := hostToContainerPath[fsstat.Mountpoint]; ok { + fsstat.ContainerPath = containerPath + } else { + for _, mountpoint := range fsstat.AllMountpoints { + if containerPath, ok := hostToContainerPath[mountpoint]; ok { + fsstat.ContainerPath = containerPath + break + } + } + } + stats.Filesystem = append(stats.Filesystem, fsstat) } } diff --git a/container/raw/handler_test.go b/container/raw/handler_test.go index 35315dfaab..2a8539da1b 100644 --- a/container/raw/handler_test.go +++ b/container/raw/handler_test.go @@ -122,6 +122,24 @@ func TestFsToFsStats(t *testing.T) { WeightedIoTime: uint64(100), }, }, + "has_mountpoint": { + fs: &fs.Fs{ + DeviceInfo: fs.DeviceInfo{Device: "/dev/sdb"}, + Mountpoint: "/var/lib/kubelet/pods/abc123/volumes/kubernetes.io~csi/my-pvc/mount", + Type: fs.VFS, + Capacity: uint64(1024 * 1024), + Free: uint64(1024), + Available: uint64(1024), + }, + expected: info.FsStats{ + Device: "/dev/sdb", + Mountpoint: "/var/lib/kubelet/pods/abc123/volumes/kubernetes.io~csi/my-pvc/mount", + Type: fs.VFS.String(), + Limit: uint64(1024 * 1024), + Usage: uint64(1024*1024) - uint64(1024), + Available: uint64(1024), + }, + }, } for testName, testCase := range testCases { actual := fsToFsStats(testCase.fs) @@ -381,6 +399,82 @@ func TestGetFsStats(t *testing.T) { }, }, }, + "random container with mountpoint and container path": { + name: "/random/container", + includedMetrics: container.MetricSet{container.DiskUsageMetrics: struct{}{}}, + externalMounts: []common.Mount{ + { + HostDir: "/var/lib/kubelet/pods/abc123/volumes/kubernetes.io~csi/my-pvc/mount", + ContainerDir: "/data", + }, + }, + getFsInfoForPath: func(mountSet map[string]struct{}) ([]fs.Fs, error) { + return []fs.Fs{{ + DeviceInfo: fs.DeviceInfo{Device: "/dev/sdb", Major: 8, Minor: 0}, + Mountpoint: "/var/lib/kubelet/pods/abc123/volumes/kubernetes.io~csi/my-pvc/mount", + Type: "ext4", + Capacity: 1000, + Free: 500, + Available: 450, + Inodes: &inodes, + InodesFree: &inodesFree, + }}, nil + }, + expectedFilesystems: []info.FsStats{{ + Device: "/dev/sdb", + Mountpoint: "/var/lib/kubelet/pods/abc123/volumes/kubernetes.io~csi/my-pvc/mount", + ContainerPath: "/data", + Type: "ext4", + Limit: 1000, + Usage: 500, + Available: 450, + HasInodes: true, + Inodes: 2000, + InodesFree: 1000, + }}, + }, + "random container with all mountpoints container path": { + name: "/random/container", + includedMetrics: container.MetricSet{container.DiskUsageMetrics: struct{}{}}, + externalMounts: []common.Mount{ + { + HostDir: "/var/lib/kubelet/pods/abc123/volumes/kubernetes.io~csi/my-pvc/mount", + ContainerDir: "/data", + }, + }, + getFsInfoForPath: func(mountSet map[string]struct{}) ([]fs.Fs, error) { + return []fs.Fs{{ + DeviceInfo: fs.DeviceInfo{Device: "/dev/sdb", Major: 8, Minor: 0}, + Mountpoint: "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc/globalmount", + AllMountpoints: []string{ + "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc/globalmount", + "/var/lib/kubelet/pods/abc123/volumes/kubernetes.io~csi/my-pvc/mount", + }, + Type: "ext4", + Capacity: 1000, + Free: 500, + Available: 450, + Inodes: &inodes, + InodesFree: &inodesFree, + }}, nil + }, + expectedFilesystems: []info.FsStats{{ + Device: "/dev/sdb", + Mountpoint: "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc/globalmount", + AllMountpoints: []string{ + "/var/lib/kubelet/plugins/kubernetes.io/csi/pv/pvc/globalmount", + "/var/lib/kubelet/pods/abc123/volumes/kubernetes.io~csi/my-pvc/mount", + }, + ContainerPath: "/data", + Type: "ext4", + Limit: 1000, + Usage: 500, + Available: 450, + HasInodes: true, + Inodes: 2000, + InodesFree: 1000, + }}, + }, "random container with disk metrics enabled": { name: "/random/container", includedMetrics: container.MetricSet{container.DiskUsageMetrics: struct{}{}, container.DiskIOMetrics: struct{}{}}, diff --git a/deploy/Dockerfile b/deploy/Dockerfile index eafe1ac436..9d111113bd 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.hub.docker.com/library/golang:1.25-alpine3.23 AS build +FROM registry.hub.docker.com/library/golang:1.25-alpine3.23@sha256:8d22e29d960bc50cd025d93d5b7c7d220b1ee9aa7a239b3c8f55a57e987e8d45 AS build # Install build depdencies for all supported arches RUN apk --no-cache add bash build-base cmake device-mapper findutils git \ @@ -49,7 +49,7 @@ RUN export GO_TAGS="libpfm,netgo"; \ fi; \ GO_FLAGS="-tags=$GO_TAGS" ./build/build.sh -FROM mirror.gcr.io/library/alpine:3.23 +FROM mirror.gcr.io/library/alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 MAINTAINER dengnan@google.com vmarmol@google.com vishnuk@google.com jimmidyson@gmail.com stclair@google.com RUN apk --no-cache add libc6-compat device-mapper findutils ndctl thin-provisioning-tools zfs && \ @@ -77,4 +77,3 @@ HEALTHCHECK --interval=30s --timeout=3s --start-period=5s \ # Use entrypoint wrapper ENTRYPOINT ["/usr/bin/entrypoint.sh"] - diff --git a/fs/fs.go b/fs/fs.go index ec2c3b2ccd..627ee31514 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -89,6 +89,9 @@ type RealFsInfo struct { dmsetup devicemapper.DmsetupClient // fsUUIDToDeviceName is a map from the filesystem UUID to its device name. fsUUIDToDeviceName map[string]string + // deviceToMountpoints maps each block device to all of its host-side mount + // paths, including bind mounts that processMounts de-duplicates away. + deviceToMountpoints map[string][]string } func NewFsInfo(context Context) (FsInfo, error) { @@ -110,12 +113,17 @@ func NewFsInfo(context Context) (FsInfo, error) { // Avoid devicemapper container mounts - these are tracked by the ThinPoolWatcher excluded := []string{fmt.Sprintf("%s/devicemapper/mnt", context.Docker.Root)} + + partitions := processMounts(mounts, excluded) + deviceToMountpoints := buildDeviceToMountpoints(mounts, excluded, partitions) + fsInfo := &RealFsInfo{ - partitions: processMounts(mounts, excluded), - labels: make(map[string]string), - mounts: make(map[string]mount.Info), - dmsetup: devicemapper.NewDmsetupClient(), - fsUUIDToDeviceName: fsUUIDToDeviceName, + partitions: partitions, + labels: make(map[string]string), + mounts: make(map[string]mount.Info), + dmsetup: devicemapper.NewDmsetupClient(), + fsUUIDToDeviceName: fsUUIDToDeviceName, + deviceToMountpoints: deviceToMountpoints, } for _, mnt := range mounts { @@ -169,12 +177,6 @@ func processMounts(mounts []*mount.Info, excludedMountpointPrefixes []string) ma partitions := make(map[string]partition) for _, mnt := range mounts { - // Use plugin system to determine if filesystem is supported - plugin := GetPluginForFsType(mnt.FSType) - if plugin == nil { - continue - } - // Avoid bind mounts, but allow tmpfs duplicates (handled by plugin's ProcessMount) if _, ok := partitions[mnt.Source]; ok { if mnt.FSType != "tmpfs" { @@ -182,20 +184,7 @@ func processMounts(mounts []*mount.Info, excludedMountpointPrefixes []string) ma } } - // Check for excluded mountpoint prefixes - hasPrefix := false - for _, prefix := range excludedMountpointPrefixes { - if strings.HasPrefix(mnt.Mountpoint, prefix) { - hasPrefix = true - break - } - } - if hasPrefix { - continue - } - - // Let plugin process the mount (handles filesystem-specific modifications) - include, processedMnt, err := plugin.ProcessMount(mnt) + include, processedMnt, err := processMountInfo(mnt, excludedMountpointPrefixes) if err != nil { klog.Warningf("error processing mount for %s: %v", mnt.FSType, err) continue @@ -215,6 +204,73 @@ func processMounts(mounts []*mount.Info, excludedMountpointPrefixes []string) ma return partitions } +func buildDeviceToMountpoints( + mounts []*mount.Info, + excludedMountpointPrefixes []string, + partitions map[string]partition, +) map[string][]string { + deviceToMountpoints := make(map[string][]string, len(partitions)) + for device, partition := range partitions { + deviceToMountpoints[device] = []string{partition.mountpoint} + } + + for _, mnt := range mounts { + include, processedMnt, err := processMountInfo(mnt, excludedMountpointPrefixes) + if err != nil { + klog.Warningf("error processing mount for %s: %v", mnt.FSType, err) + continue + } + if !include { + continue + } + if _, ok := partitions[processedMnt.Source]; !ok { + continue + } + appendUniqueMountpoint(deviceToMountpoints, processedMnt.Source, processedMnt.Mountpoint) + } + + return deviceToMountpoints +} + +func processMountInfo( + mnt *mount.Info, + excludedMountpointPrefixes []string, +) (bool, *mount.Info, error) { + plugin := GetPluginForFsType(mnt.FSType) + if plugin == nil { + return false, nil, nil + } + if isExcludedMountpoint(mnt.Mountpoint, excludedMountpointPrefixes) { + return false, nil, nil + } + include, processedMnt, err := plugin.ProcessMount(mnt) + if err != nil { + return false, nil, err + } + if !include { + return false, nil, nil + } + return true, processedMnt, nil +} + +func isExcludedMountpoint(mountpoint string, excludedMountpointPrefixes []string) bool { + for _, prefix := range excludedMountpointPrefixes { + if strings.HasPrefix(mountpoint, prefix) { + return true + } + } + return false +} + +func appendUniqueMountpoint(deviceToMountpoints map[string][]string, device, mountpoint string) { + for _, existing := range deviceToMountpoints[device] { + if existing == mountpoint { + return + } + } + deviceToMountpoints[device] = append(deviceToMountpoints[device], mountpoint) +} + // getDockerDeviceMapperInfo returns information about the devicemapper device and "partition" if // docker is using devicemapper for its storage driver. If a loopback device is being used, don't // return any information or error, as we want to report based on the actual partition where the @@ -429,6 +485,8 @@ func (i *RealFsInfo) GetFsInfoForPath(mountSet map[string]struct{}) ([]Fs, error Major: uint(partition.major), Minor: uint(partition.minor), } + fs.Mountpoint = partition.mountpoint + fs.AllMountpoints = i.deviceToMountpoints[device] if val, ok := diskStatsMap[device]; ok { fs.DiskStats = val } else { @@ -483,6 +541,8 @@ func (i *RealFsInfo) GetFsInfoForPath(mountSet map[string]struct{}) ([]Fs, error Major: uint(partition.major), Minor: uint(partition.minor), } + fs.Mountpoint = partition.mountpoint + fs.AllMountpoints = i.deviceToMountpoints[device] if val, ok := diskStatsMap[device]; ok { fs.DiskStats = val diff --git a/fs/fs_test.go b/fs/fs_test.go index 1db4ceca9c..b75f9f2b98 100644 --- a/fs/fs_test.go +++ b/fs/fs_test.go @@ -730,3 +730,28 @@ func TestProcessMounts(t *testing.T) { } } } + +func TestBuildDeviceToMountpoints(t *testing.T) { + mounts := []*mount.Info{ + {Root: "/", Mountpoint: "/var/lib/containers/storage/overlay", Source: "overlay", FSType: "overlay", Major: 253, Minor: 5}, + {Root: "/", Mountpoint: "/var/lib/kubelet/pods/abc/volumes/kubernetes.io~csi/pvc/mount", Source: "overlay", FSType: "overlay", Major: 253, Minor: 5}, + {Root: "/", Mountpoint: "/tmpfs-primary", Source: "tmpfs", FSType: "tmpfs", Major: 253, Minor: 4}, + {Root: "/", Mountpoint: "/tmpfs-secondary", Source: "tmpfs", FSType: "tmpfs", Major: 253, Minor: 4}, + {Root: "/", Mountpoint: "/exclude/me", Source: "/dev/sda1", FSType: "xfs", Major: 253, Minor: 1}, + } + excludedPrefixes := []string{"/exclude"} + + partitions := processMounts(mounts, excludedPrefixes) + actual := buildDeviceToMountpoints(mounts, excludedPrefixes, partitions) + + expected := map[string][]string{ + "overlay_253-5": { + "/var/lib/kubelet/pods/abc/volumes/kubernetes.io~csi/pvc/mount", + "/var/lib/containers/storage/overlay", + }, + "/tmpfs-primary": {"/tmpfs-primary"}, + "/tmpfs-secondary": {"/tmpfs-secondary"}, + } + + assert.Equal(t, expected, actual) +} diff --git a/fs/types.go b/fs/types.go index 6bc91985cb..424a548738 100644 --- a/fs/types.go +++ b/fs/types.go @@ -71,6 +71,11 @@ type Fs struct { Inodes *uint64 InodesFree *uint64 DiskStats DiskStats + // Mountpoint is the primary host-side path where this device is mounted. + Mountpoint string + // AllMountpoints lists every host-side path where this device is mounted, + // including bind mounts (e.g. CSI plugin globalmount + per-pod kubelet path). + AllMountpoints []string } type DiskStats struct { diff --git a/go.mod b/go.mod index f6fa4fa359..9ef8c3a764 100644 --- a/go.mod +++ b/go.mod @@ -70,9 +70,9 @@ require ( github.com/stretchr/objx v0.5.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.39.0 // indirect - go.opentelemetry.io/otel/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect golang.org/x/net v0.48.0 // indirect golang.org/x/text v0.32.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect diff --git a/go.sum b/go.sum index b8a9e3325a..7f96b11b3b 100644 --- a/go.sum +++ b/go.sum @@ -136,16 +136,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/info/v1/container.go b/info/v1/container.go index bb9e8c87d0..52cfc3b3b5 100644 --- a/info/v1/container.go +++ b/info/v1/container.go @@ -877,6 +877,18 @@ type FsStats struct { // last update of this field. This can provide an easy measure of both // I/O completion time and the backlog that may be accumulating. WeightedIoTime uint64 `json:"weighted_io_time"` + + // Mountpoint is the host-side path where this filesystem device is mounted. + Mountpoint string `json:"mountpoint,omitempty"` + + // AllMountpoints lists every host-side path where this device is mounted, + // including bind mounts (e.g. both the CSI plugin globalmount path and the + // per-pod kubelet volume path). Mountpoint is always present in this slice. + AllMountpoints []string `json:"all_mountpoints,omitempty"` + + // ContainerPath is the path inside the container corresponding to this + // filesystem mount (populated from external mounts / container hints config). + ContainerPath string `json:"container_path,omitempty"` } type AcceleratorStats struct { diff --git a/integration/tests/api/event_test.go b/integration/tests/api/event_test.go index 1b873292d3..d5e040e94d 100644 --- a/integration/tests/api/event_test.go +++ b/integration/tests/api/event_test.go @@ -88,7 +88,7 @@ func TestOomKillEventConstraint(t *testing.T) { containerID := fm.Docker().Run( framework.DockerRunArgs{ Image: "registry.k8s.io/busybox:1.27", - Args: []string{"-m=" + tt.memLimit}, + Args: []string{"-m=" + tt.memLimit, "--memory-swap=" + tt.memLimit}, }, "sh", "-c", "a=1; while true; do a=\"${a}${a}\"; done", ) diff --git a/manager/container_test.go b/manager/container_test.go index f4f68f8185..d92888feb6 100644 --- a/manager/container_test.go +++ b/manager/container_test.go @@ -573,6 +573,22 @@ func TestNextHousekeepingInterval(t *testing.T) { expectedMin: base, expectedMax: base, }, + { + name: "negative splay factor resets to default", + splayFactor: -1.0, + jitterFactor: 0.5, + firstHousekeeping: true, + expectedMin: base, + expectedMax: base + time.Duration(jitterDefault*float64(base)), + }, + { + name: "negative jitter factor resets to default", + splayFactor: 1.0, + jitterFactor: -0.5, + firstHousekeeping: false, + expectedMin: base, + expectedMax: base + time.Duration(jitterDefault*float64(base)), + }, } for _, tc := range tests { diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 57effd32ef..4957dd89f8 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -18,6 +18,7 @@ import ( "fmt" "regexp" "strconv" + "strings" "time" "github.com/google/cadvisor/container" @@ -43,9 +44,20 @@ func asNanosecondsToSeconds(v uint64) float64 { return float64(v) / 1e9 } -// fsValues is a helper method for assembling per-filesystem stats. -func fsValues(fsStats []info.FsStats, valueFn func(*info.FsStats) float64, timestamp time.Time) metricValues { - values := make(metricValues, 0, len(fsStats)) +var kubeletVolumePathRegexp = regexp.MustCompile(`/pods/[^/]+/volumes/([^/]+)/([^/]+)`) + +// ioValues is a helper method for assembling per-disk and per-filesystem stats. +func ioValues(ioStats []info.PerDiskStats, ioType string, ioValueFn func(uint64) float64, + fsStats []info.FsStats, valueFn func(*info.FsStats) float64, timestamp time.Time) metricValues { + + values := make(metricValues, 0, len(ioStats)+len(fsStats)) + for _, stat := range ioStats { + values = append(values, metricValue{ + value: ioValueFn(stat.Stats[ioType]), + labels: []string{stat.Device}, + timestamp: timestamp, + }) + } for _, stat := range fsStats { values = append(values, metricValue{ value: valueFn(&stat), @@ -56,22 +68,120 @@ func fsValues(fsStats []info.FsStats, valueFn func(*info.FsStats) float64, times return values } -// ioValues is a helper method for assembling per-disk and per-filesystem stats. -func ioValues(ioStats []info.PerDiskStats, ioType string, ioValueFn func(uint64) float64, - fsStats []info.FsStats, valueFn func(*info.FsStats) float64, timestamp time.Time) metricValues { +type volumePath struct { + volumeName string + volumeType string +} + +// parseMountPointIntoVolumePath extracts the volume name from a kubelet volume mount +// path. For CSI volumes this is the PersistentVolumeClaim name. Returns an +// empty string for paths that do not match the standard kubelet volume path. +// +// Example: +// +// /var/lib/kubelet/pods/abc123/volumes/kubernetes.io~csi/my-pvc/mount → "my-pvc" +func parseMountPointIntoVolumePath(path string) volumePath { + // /var/lib/kubelet/pods//volumes//[/…] + // + // Capture group 1 is the volume type, and capture group 2 is the volume name, + // which for CSI volumes corresponds to the PersistentVolumeClaim name. + m := kubeletVolumePathRegexp.FindStringSubmatch(path) + if m != nil { + return volumePath{ + volumeName: m[2], + volumeType: m[1], + } + } + + return volumePath{} +} +// deviceFsInfo holds the filesystem identity labels for a block device. +type deviceFsInfo struct { + hostMountpoint string + containerMountpath string + volumeName string + volumeType string +} + +// bestMountpoint selects the most informative host-side mountpoint from fs.AllMountpoints. +// It prefers paths containing "/pods/" (the kubelet per-pod volume path) since those +// can be decoded into a PVC name via parseMountPointIntoVolumePath. +func bestMountpoint(fs *info.FsStats) string { + for _, mp := range fs.AllMountpoints { + if strings.Contains(mp, "/pods/") { + return mp + } + } + if len(fs.AllMountpoints) > 0 { + return fs.AllMountpoints[0] + } + return fs.Mountpoint +} + +// buildDeviceFsMap builds a map from device name to filesystem identity labels +// by inspecting the container's filesystem stats. For the aggregator to correlate +// disk IO with PVCs, the kubelet per-pod volume path is preferred over globalmount. +func buildDeviceFsMap(fsStats []info.FsStats) map[string]deviceFsInfo { + m := make(map[string]deviceFsInfo, len(fsStats)) + for i := range fsStats { + fs := &fsStats[i] + if fs.Device == "" { + continue + } + mp := bestMountpoint(fs) + vp := parseMountPointIntoVolumePath(mp) + m[fs.Device] = deviceFsInfo{ + hostMountpoint: mp, + containerMountpath: fs.ContainerPath, + volumeName: vp.volumeName, + volumeType: vp.volumeType, + } + } + return m +} + +// fsValuesEnriched is like fsValues but appends host_mountpoint, container_mountpath, +// volume_name, and volume_type labels derived from the filesystem info. +func fsValuesEnriched(fsStats []info.FsStats, valueFn func(*info.FsStats) float64, timestamp time.Time) metricValues { + values := make(metricValues, 0, len(fsStats)) + for i := range fsStats { + fs := &fsStats[i] + mp := bestMountpoint(fs) + vp := parseMountPointIntoVolumePath(mp) + values = append(values, metricValue{ + value: valueFn(fs), + labels: []string{fs.Device, mp, fs.ContainerPath, vp.volumeName, vp.volumeType}, + timestamp: timestamp, + }) + } + return values +} + +// ioValuesEnriched is like ioValues but appends host_mountpoint, container_mountpath, +// volume_name, and volume_type labels. For ioStats entries the device is looked up +// in fsMap; for fsStats entries the info is read directly. +func ioValuesEnriched( + ioStats []info.PerDiskStats, ioType string, ioValueFn func(uint64) float64, + fsStats []info.FsStats, valueFn func(*info.FsStats) float64, + fsMap map[string]deviceFsInfo, timestamp time.Time, +) metricValues { values := make(metricValues, 0, len(ioStats)+len(fsStats)) for _, stat := range ioStats { + fi := fsMap[stat.Device] values = append(values, metricValue{ value: ioValueFn(stat.Stats[ioType]), - labels: []string{stat.Device}, + labels: []string{stat.Device, fi.hostMountpoint, fi.containerMountpath, fi.volumeName, fi.volumeType}, timestamp: timestamp, }) } - for _, stat := range fsStats { + for i := range fsStats { + fs := &fsStats[i] + mp := bestMountpoint(fs) + vp := parseMountPointIntoVolumePath(mp) values = append(values, metricValue{ - value: valueFn(&stat), - labels: []string{stat.Device}, + value: valueFn(fs), + labels: []string{fs.Device, mp, fs.ContainerPath, vp.volumeName, vp.volumeType}, timestamp: timestamp, }) } @@ -578,9 +688,9 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri name: "container_fs_inodes_free", help: "Number of available Inodes", valueType: prometheus.GaugeValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return fsValues(s.Filesystem, func(fs *info.FsStats) float64 { + return fsValuesEnriched(s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.InodesFree) }, s.Timestamp) }, @@ -588,9 +698,9 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri name: "container_fs_inodes_total", help: "Number of Inodes", valueType: prometheus.GaugeValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return fsValues(s.Filesystem, func(fs *info.FsStats) float64 { + return fsValuesEnriched(s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.Inodes) }, s.Timestamp) }, @@ -598,9 +708,9 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri name: "container_fs_limit_bytes", help: "Number of bytes that can be consumed by the container on this filesystem.", valueType: prometheus.GaugeValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return fsValues(s.Filesystem, func(fs *info.FsStats) float64 { + return fsValuesEnriched(s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.Limit) }, s.Timestamp) }, @@ -608,9 +718,9 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri name: "container_fs_usage_bytes", help: "Number of bytes that are consumed by the container on this filesystem.", valueType: prometheus.GaugeValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return fsValues(s.Filesystem, func(fs *info.FsStats) float64 { + return fsValuesEnriched(s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.Usage) }, s.Timestamp) }, @@ -623,173 +733,185 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri name: "container_fs_reads_bytes_total", help: "Cumulative count of bytes read", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoServiceBytes, "Read", asFloat64, nil, nil, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_reads_total", help: "Cumulative count of reads completed", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoServiced, "Read", asFloat64, s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.ReadsCompleted) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_sector_reads_total", help: "Cumulative count of sector reads completed", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.Sectors, "Read", asFloat64, s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.SectorsRead) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_reads_merged_total", help: "Cumulative count of reads merged", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoMerged, "Read", asFloat64, s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.ReadsMerged) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_read_seconds_total", help: "Cumulative count of seconds spent reading", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoServiceTime, "Read", asNanosecondsToSeconds, s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.ReadTime) / float64(time.Second) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_writes_bytes_total", help: "Cumulative count of bytes written", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoServiceBytes, "Write", asFloat64, nil, nil, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_writes_total", help: "Cumulative count of writes completed", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoServiced, "Write", asFloat64, s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.WritesCompleted) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_sector_writes_total", help: "Cumulative count of sector writes completed", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.Sectors, "Write", asFloat64, s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.SectorsWritten) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_writes_merged_total", help: "Cumulative count of writes merged", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoMerged, "Write", asFloat64, s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.WritesMerged) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_write_seconds_total", help: "Cumulative count of seconds spent writing", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoServiceTime, "Write", asNanosecondsToSeconds, s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.WriteTime) / float64(time.Second) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_io_current", help: "Number of I/Os currently in progress", valueType: prometheus.GaugeValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoQueued, "Total", asFloat64, s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.IoInProgress) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_io_time_seconds_total", help: "Cumulative count of seconds spent doing I/Os", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoServiceTime, "Total", asNanosecondsToSeconds, s.Filesystem, func(fs *info.FsStats) float64 { return float64(float64(fs.IoTime) / float64(time.Second)) }, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_io_time_weighted_seconds_total", help: "Cumulative weighted I/O time in seconds", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return fsValues(s.Filesystem, func(fs *info.FsStats) float64 { + return fsValuesEnriched(s.Filesystem, func(fs *info.FsStats) float64 { return float64(fs.WeightedIoTime) / float64(time.Second) }, s.Timestamp) }, @@ -797,48 +919,52 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri name: "container_fs_io_cost_usage_seconds_total", help: "Cumulative IOCost usage in seconds", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoCostUsage, "Count", asMicrosecondsToSeconds, []info.FsStats{}, nil, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_io_cost_wait_seconds_total", help: "Cumulative IOCost wait in seconds", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoCostWait, "Count", asMicrosecondsToSeconds, []info.FsStats{}, nil, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_io_cost_indebt_seconds_total", help: "Cumulative IOCost debt in seconds", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoCostIndebt, "Count", asMicrosecondsToSeconds, []info.FsStats{}, nil, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, { name: "container_fs_io_cost_indelay_seconds_total", help: "Cumulative IOCost delay in seconds", valueType: prometheus.CounterValue, - extraLabels: []string{"device"}, + extraLabels: []string{"device", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { - return ioValues( + fsMap := buildDeviceFsMap(s.Filesystem) + return ioValuesEnriched( s.DiskIo.IoCostIndelay, "Count", asMicrosecondsToSeconds, []info.FsStats{}, nil, - s.Timestamp, + fsMap, s.Timestamp, ) }, }, @@ -846,17 +972,25 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri name: "container_blkio_device_usage_total", help: "Blkio Device bytes usage", valueType: prometheus.CounterValue, - extraLabels: []string{"device", "major", "minor", "operation"}, + extraLabels: []string{"device", "major", "minor", "operation", "host_mountpoint", "container_mountpath", "volume_name", "volume_type"}, getValues: func(s *info.ContainerStats) metricValues { + fsMap := buildDeviceFsMap(s.Filesystem) var values metricValues for _, diskStat := range s.DiskIo.IoServiceBytes { + fi := fsMap[diskStat.Device] for operation, value := range diskStat.Stats { values = append(values, metricValue{ value: float64(value), - labels: []string{diskStat.Device, + labels: []string{ + diskStat.Device, strconv.Itoa(int(diskStat.Major)), strconv.Itoa(int(diskStat.Minor)), - operation}, + operation, + fi.hostMountpoint, + fi.containerMountpath, + fi.volumeName, + fi.volumeType, + }, timestamp: s.Timestamp, }) } diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index 928fbf1674..b1f08ec474 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -16,6 +16,7 @@ package metrics import ( "errors" + "flag" "os" "testing" "time" @@ -26,6 +27,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/assert" clock "k8s.io/utils/clock/testing" ) @@ -75,7 +77,28 @@ func TestPrometheusCollectorWithPerfAggregated(t *testing.T) { testPrometheusCollector(t, reg, "testdata/prometheus_metrics_perf_aggregated") } +var update = flag.Bool("update", false, "update golden fixture files") + func testPrometheusCollector(t *testing.T, gatherer prometheus.Gatherer, metricsFile string) { + if *update { + f, err := os.Create(metricsFile) + if err != nil { + t.Fatalf("failed to create fixture %s: %s", metricsFile, err) + } + defer f.Close() + mfs, err := gatherer.Gather() + if err != nil { + t.Fatalf("failed to gather metrics: %s", err) + } + enc := expfmt.NewEncoder(f, expfmt.NewFormat(expfmt.TypeTextPlain)) + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + t.Fatalf("failed to encode metric family: %s", err) + } + } + return + } + wantMetrics, err := os.Open(metricsFile) if err != nil { t.Fatalf("unable to read input test file %s", metricsFile) @@ -464,3 +487,47 @@ func TestCPUBurstMetrics(t *testing.T) { }) } } + +func TestParseMountPointIntoVolumePath(t *testing.T) { + testCases := []struct { + name string + path string + expected string + }{ + { + name: "standard CSI volume path", + path: "/var/lib/kubelet/pods/abc123-def456/volumes/kubernetes.io~csi/my-database-pvc/mount", + expected: "my-database-pvc", + }, + { + name: "NFS volume path", + path: "/var/lib/kubelet/pods/abc/volumes/kubernetes.io~nfs/nfs-vol/mount", + expected: "nfs-vol", + }, + { + name: "non-standard kubelet root dir", + path: "/data/kubelet/pods/abc/volumes/kubernetes.io~csi/my-pvc/mount", + expected: "my-pvc", + }, + { + name: "no match - plain host path", + path: "/mnt/data", + expected: "", + }, + { + name: "empty string", + path: "", + expected: "", + }, + { + name: "root cgroup path", + path: "/", + expected: "", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, parseMountPointIntoVolumePath(tc.path).volumeName) + }) + } +} diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index 5be1aa7f78..1f37ba03fc 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -3,12 +3,18 @@ cadvisor_version_info{cadvisorRevision="abcdef",cadvisorVersion="0.16.0",dockerVersion="1.8.1",kernelVersion="4.1.6-200.fc22.x86_64",osVersion="Fedora 22 (Twenty Two)"} 1 # HELP container_blkio_device_usage_total Blkio Device bytes usage # TYPE container_blkio_device_usage_total counter -container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Async",zone_name="hello"} 1 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Discard",zone_name="hello"} 2 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Read",zone_name="hello"} 3 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Sync",zone_name="hello"} 4 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Total",zone_name="hello"} 5 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Write",zone_name="hello"} 6 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Async",volume_name="",volume_type="",zone_name="hello"} 1 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Discard",volume_name="",volume_type="",zone_name="hello"} 2 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Read",volume_name="",volume_type="",zone_name="hello"} 3 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Sync",volume_name="",volume_type="",zone_name="hello"} 4 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Total",volume_name="",volume_type="",zone_name="hello"} 5 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Write",volume_name="",volume_type="",zone_name="hello"} 6 1395066363000 +# HELP container_cpu_cfs_burst_periods_total Number of periods when burst occurs. +# TYPE container_cpu_cfs_burst_periods_total counter +container_cpu_cfs_burst_periods_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 1395066363000 +# HELP container_cpu_cfs_burst_seconds_total Total time duration the container has been bursted. +# TYPE container_cpu_cfs_burst_seconds_total counter +container_cpu_cfs_burst_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.5 1395066363000 # HELP container_cpu_cfs_periods_total Number of elapsed enforcement period intervals. # TYPE container_cpu_cfs_periods_total counter container_cpu_cfs_periods_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 723 1395066363000 @@ -18,12 +24,6 @@ container_cpu_cfs_throttled_periods_total{container_env_foo_env="prod",container # HELP container_cpu_cfs_throttled_seconds_total Total time duration the container has been throttled. # TYPE container_cpu_cfs_throttled_seconds_total counter container_cpu_cfs_throttled_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.724314 1395066363000 -# HELP container_cpu_cfs_burst_periods_total Number of periods when burst occurs. -# TYPE container_cpu_cfs_burst_periods_total counter -container_cpu_cfs_burst_periods_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 1395066363000 -# HELP container_cpu_cfs_burst_seconds_total Total time duration the container has been bursted. -# TYPE container_cpu_cfs_burst_seconds_total counter -container_cpu_cfs_burst_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.5 1395066363000 # HELP container_cpu_load_average_10s Value of container cpu load average over the last 10 seconds. # TYPE container_cpu_load_average_10s gauge container_cpu_load_average_10s{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2 1395066363000 @@ -66,82 +66,82 @@ container_custom_app_metric_3{app_test_label="test_value",container_env_foo_env= container_file_descriptors{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 # HELP container_fs_inodes_free Number of available Inodes # TYPE container_fs_inodes_free gauge -container_fs_inodes_free{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 524288 1395066363000 -container_fs_inodes_free{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 262144 1395066363000 +container_fs_inodes_free{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 524288 1395066363000 +container_fs_inodes_free{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 262144 1395066363000 # HELP container_fs_inodes_total Number of Inodes # TYPE container_fs_inodes_total gauge -container_fs_inodes_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.097152e+06 1395066363000 -container_fs_inodes_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.097152e+06 1395066363000 +container_fs_inodes_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 2.097152e+06 1395066363000 +container_fs_inodes_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 2.097152e+06 1395066363000 +# HELP container_fs_io_cost_indebt_seconds_total Cumulative IOCost debt in seconds +# TYPE container_fs_io_cost_indebt_seconds_total counter +container_fs_io_cost_indebt_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 0.5 1395066363000 +# HELP container_fs_io_cost_indelay_seconds_total Cumulative IOCost delay in seconds +# TYPE container_fs_io_cost_indelay_seconds_total counter +container_fs_io_cost_indelay_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 0.75 1395066363000 +# HELP container_fs_io_cost_usage_seconds_total Cumulative IOCost usage in seconds +# TYPE container_fs_io_cost_usage_seconds_total counter +container_fs_io_cost_usage_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 1.5 1395066363000 +# HELP container_fs_io_cost_wait_seconds_total Cumulative IOCost wait in seconds +# TYPE container_fs_io_cost_wait_seconds_total counter +container_fs_io_cost_wait_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 2.5 1395066363000 # HELP container_fs_io_current Number of I/Os currently in progress # TYPE container_fs_io_current gauge -container_fs_io_current{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 1395066363000 -container_fs_io_current{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 47 1395066363000 +container_fs_io_current{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 42 1395066363000 +container_fs_io_current{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 47 1395066363000 # HELP container_fs_io_time_seconds_total Cumulative count of seconds spent doing I/Os # TYPE container_fs_io_time_seconds_total counter -container_fs_io_time_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.3e-08 1395066363000 -container_fs_io_time_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.8e-08 1395066363000 +container_fs_io_time_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.3e-08 1395066363000 +container_fs_io_time_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.8e-08 1395066363000 # HELP container_fs_io_time_weighted_seconds_total Cumulative weighted I/O time in seconds # TYPE container_fs_io_time_weighted_seconds_total counter -container_fs_io_time_weighted_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.4e-08 1395066363000 -container_fs_io_time_weighted_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.9e-08 1395066363000 -# HELP container_fs_io_cost_usage_seconds_total Cumulative IOCost usage in seconds -# TYPE container_fs_io_cost_usage_seconds_total counter -container_fs_io_cost_usage_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.5 1395066363000 -# HELP container_fs_io_cost_wait_seconds_total Cumulative IOCost wait in seconds -# TYPE container_fs_io_cost_wait_seconds_total counter -container_fs_io_cost_wait_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.5 1395066363000 -# HELP container_fs_io_cost_indebt_seconds_total Cumulative IOCost debt in seconds -# TYPE container_fs_io_cost_indebt_seconds_total counter -container_fs_io_cost_indebt_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.5 1395066363000 -# HELP container_fs_io_cost_indelay_seconds_total Cumulative IOCost delay in seconds -# TYPE container_fs_io_cost_indelay_seconds_total counter -container_fs_io_cost_indelay_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.75 1395066363000 +container_fs_io_time_weighted_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.4e-08 1395066363000 +container_fs_io_time_weighted_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.9e-08 1395066363000 # HELP container_fs_limit_bytes Number of bytes that can be consumed by the container on this filesystem. # TYPE container_fs_limit_bytes gauge -container_fs_limit_bytes{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 22 1395066363000 -container_fs_limit_bytes{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 37 1395066363000 +container_fs_limit_bytes{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 22 1395066363000 +container_fs_limit_bytes{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 37 1395066363000 # HELP container_fs_read_seconds_total Cumulative count of seconds spent reading # TYPE container_fs_read_seconds_total counter -container_fs_read_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.7e-08 1395066363000 -container_fs_read_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.2e-08 1395066363000 +container_fs_read_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 2.7e-08 1395066363000 +container_fs_read_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.2e-08 1395066363000 # HELP container_fs_reads_bytes_total Cumulative count of bytes read # TYPE container_fs_reads_bytes_total counter -container_fs_reads_bytes_total{container_env_foo_env="prod",container_label_foo_label="bar",device="/dev/sdb",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000 +container_fs_reads_bytes_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 3 1395066363000 # HELP container_fs_reads_merged_total Cumulative count of reads merged # TYPE container_fs_reads_merged_total counter -container_fs_reads_merged_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 1395066363000 -container_fs_reads_merged_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 1395066363000 +container_fs_reads_merged_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 25 1395066363000 +container_fs_reads_merged_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 40 1395066363000 # HELP container_fs_reads_total Cumulative count of reads completed # TYPE container_fs_reads_total counter -container_fs_reads_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 24 1395066363000 -container_fs_reads_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 1395066363000 +container_fs_reads_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 24 1395066363000 +container_fs_reads_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 39 1395066363000 # HELP container_fs_sector_reads_total Cumulative count of sector reads completed # TYPE container_fs_sector_reads_total counter -container_fs_sector_reads_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 26 1395066363000 -container_fs_sector_reads_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 41 1395066363000 +container_fs_sector_reads_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 26 1395066363000 +container_fs_sector_reads_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 41 1395066363000 # HELP container_fs_sector_writes_total Cumulative count of sector writes completed # TYPE container_fs_sector_writes_total counter -container_fs_sector_writes_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 1395066363000 -container_fs_sector_writes_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 45 1395066363000 +container_fs_sector_writes_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 40 1395066363000 +container_fs_sector_writes_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 45 1395066363000 # HELP container_fs_usage_bytes Number of bytes that are consumed by the container on this filesystem. # TYPE container_fs_usage_bytes gauge -container_fs_usage_bytes{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 23 1395066363000 -container_fs_usage_bytes{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 38 1395066363000 +container_fs_usage_bytes{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 23 1395066363000 +container_fs_usage_bytes{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 38 1395066363000 # HELP container_fs_write_seconds_total Cumulative count of seconds spent writing # TYPE container_fs_write_seconds_total counter -container_fs_write_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.1e-08 1395066363000 -container_fs_write_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.6e-08 1395066363000 +container_fs_write_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.1e-08 1395066363000 +container_fs_write_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.6e-08 1395066363000 # HELP container_fs_writes_bytes_total Cumulative count of bytes written # TYPE container_fs_writes_bytes_total counter -container_fs_writes_bytes_total{container_env_foo_env="prod",container_label_foo_label="bar",device="/dev/sdb",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 6 1395066363000 +container_fs_writes_bytes_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 6 1395066363000 # HELP container_fs_writes_merged_total Cumulative count of writes merged # TYPE container_fs_writes_merged_total counter -container_fs_writes_merged_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 1395066363000 -container_fs_writes_merged_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 44 1395066363000 +container_fs_writes_merged_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 39 1395066363000 +container_fs_writes_merged_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 44 1395066363000 # HELP container_fs_writes_total Cumulative count of writes completed # TYPE container_fs_writes_total counter -container_fs_writes_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 28 1395066363000 -container_fs_writes_total{container_env_foo_env="prod",container_label_foo_label="bar",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 43 1395066363000 +container_fs_writes_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 28 1395066363000 +container_fs_writes_total{container_env_foo_env="prod",container_label_foo_label="bar",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 43 1395066363000 # HELP container_health_state The result of the container's health check # TYPE container_health_state gauge container_health_state{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000 @@ -160,6 +160,18 @@ container_hugetlb_usage_bytes{container_env_foo_env="prod",container_label_foo_l # HELP container_last_seen Last time a container was seen by the exporter # TYPE container_last_seen gauge container_last_seen{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.395066363e+09 1395066363000 +# HELP container_llc_occupancy_bytes Last level cache usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). +# TYPE container_llc_occupancy_bytes gauge +container_llc_occupancy_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 162626 1395066363000 +container_llc_occupancy_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 213777 1395066363000 +# HELP container_memory_bandwidth_bytes Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). +# TYPE container_memory_bandwidth_bytes gauge +container_memory_bandwidth_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 4.512312e+06 1395066363000 +container_memory_bandwidth_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 2.173713e+06 1395066363000 +# HELP container_memory_bandwidth_local_bytes Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). +# TYPE container_memory_bandwidth_local_bytes gauge +container_memory_bandwidth_local_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 2.390393e+06 1395066363000 +container_memory_bandwidth_local_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 1.231233e+06 1395066363000 # HELP container_memory_cache Number of bytes of page cache memory. # TYPE container_memory_cache gauge container_memory_cache{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000 @@ -388,26 +400,26 @@ container_network_udp_usage_total{container_env_foo_env="prod",container_label_f # HELP container_oom_events_total Count of out of memory events observed for the container # TYPE container_oom_events_total counter container_oom_events_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000 -# HELP container_perf_events_total Perf event metric. -# TYPE container_perf_events_total counter -container_perf_events_total{container_env_foo_env="prod",container_label_foo_label="bar",cpu="0",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 123 1395066363000 -container_perf_events_total{container_env_foo_env="prod",container_label_foo_label="bar",cpu="0",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 321 1395066363000 -container_perf_events_total{container_env_foo_env="prod",container_label_foo_label="bar",cpu="1",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 456 1395066363000 -container_perf_events_total{container_env_foo_env="prod",container_label_foo_label="bar",cpu="1",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 789 1395066363000 # HELP container_perf_events_scaling_ratio Perf event metric scaling ratio. # TYPE container_perf_events_scaling_ratio gauge container_perf_events_scaling_ratio{container_env_foo_env="prod",container_label_foo_label="bar",cpu="0",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000 container_perf_events_scaling_ratio{container_env_foo_env="prod",container_label_foo_label="bar",cpu="0",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.66666666666 1395066363000 container_perf_events_scaling_ratio{container_env_foo_env="prod",container_label_foo_label="bar",cpu="1",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.5 1395066363000 container_perf_events_scaling_ratio{container_env_foo_env="prod",container_label_foo_label="bar",cpu="1",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.33333333333 1395066363000 -# HELP container_perf_uncore_events_total Perf uncore event metric. -# TYPE container_perf_uncore_events_total counter -container_perf_uncore_events_total{container_env_foo_env="prod",container_label_foo_label="bar",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="0",zone_name="hello"} 1.231231512e+09 1395066363000 -container_perf_uncore_events_total{container_env_foo_env="prod",container_label_foo_label="bar",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="1",zone_name="hello"} 1.111231331e+09 1395066363000 +# HELP container_perf_events_total Perf event metric. +# TYPE container_perf_events_total counter +container_perf_events_total{container_env_foo_env="prod",container_label_foo_label="bar",cpu="0",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 123 1395066363000 +container_perf_events_total{container_env_foo_env="prod",container_label_foo_label="bar",cpu="0",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 321 1395066363000 +container_perf_events_total{container_env_foo_env="prod",container_label_foo_label="bar",cpu="1",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 456 1395066363000 +container_perf_events_total{container_env_foo_env="prod",container_label_foo_label="bar",cpu="1",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 789 1395066363000 # HELP container_perf_uncore_events_scaling_ratio Perf uncore event metric scaling ratio. # TYPE container_perf_uncore_events_scaling_ratio gauge container_perf_uncore_events_scaling_ratio{container_env_foo_env="prod",container_label_foo_label="bar",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="0",zone_name="hello"} 1 1395066363000 container_perf_uncore_events_scaling_ratio{container_env_foo_env="prod",container_label_foo_label="bar",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="1",zone_name="hello"} 1 1395066363000 +# HELP container_perf_uncore_events_total Perf uncore event metric. +# TYPE container_perf_uncore_events_total counter +container_perf_uncore_events_total{container_env_foo_env="prod",container_label_foo_label="bar",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="0",zone_name="hello"} 1.231231512e+09 1395066363000 +container_perf_uncore_events_total{container_env_foo_env="prod",container_label_foo_label="bar",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="1",zone_name="hello"} 1.111231331e+09 1395066363000 # HELP container_pressure_cpu_stalled_seconds_total Total time duration no tasks in the container could make progress due to CPU congestion. # TYPE container_pressure_cpu_stalled_seconds_total counter container_pressure_cpu_stalled_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.0001 1395066363000 @@ -469,15 +481,3 @@ container_threads_max{container_env_foo_env="prod",container_label_foo_label="ba # HELP container_ulimits_soft Soft ulimit values for the container root process. Unlimited if -1, except priority and nice # TYPE container_ulimits_soft gauge container_ulimits_soft{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",ulimit="max_open_files",zone_name="hello"} 16384 1395066363000 -# HELP container_llc_occupancy_bytes Last level cache usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). -# TYPE container_llc_occupancy_bytes gauge -container_llc_occupancy_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 162626 1395066363000 -container_llc_occupancy_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 213777 1395066363000 -# HELP container_memory_bandwidth_bytes Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). -# TYPE container_memory_bandwidth_bytes gauge -container_memory_bandwidth_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 4.512312e+06 1395066363000 -container_memory_bandwidth_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 2.173713e+06 1395066363000 -# HELP container_memory_bandwidth_local_bytes Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). -# TYPE container_memory_bandwidth_local_bytes gauge -container_memory_bandwidth_local_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 2.390393e+06 1395066363000 -container_memory_bandwidth_local_bytes{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 1.231233e+06 1395066363000 diff --git a/metrics/testdata/prometheus_metrics_whitelist_filtered b/metrics/testdata/prometheus_metrics_whitelist_filtered index 68206a2274..e7ce19866b 100644 --- a/metrics/testdata/prometheus_metrics_whitelist_filtered +++ b/metrics/testdata/prometheus_metrics_whitelist_filtered @@ -3,12 +3,18 @@ cadvisor_version_info{cadvisorRevision="abcdef",cadvisorVersion="0.16.0",dockerVersion="1.8.1",kernelVersion="4.1.6-200.fc22.x86_64",osVersion="Fedora 22 (Twenty Two)"} 1 # HELP container_blkio_device_usage_total Blkio Device bytes usage # TYPE container_blkio_device_usage_total counter -container_blkio_device_usage_total{container_env_foo_env="prod",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Async",zone_name="hello"} 1 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Discard",zone_name="hello"} 2 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Read",zone_name="hello"} 3 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Sync",zone_name="hello"} 4 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Total",zone_name="hello"} 5 1395066363000 -container_blkio_device_usage_total{container_env_foo_env="prod",device="/dev/sdb",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Write",zone_name="hello"} 6 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Async",volume_name="",volume_type="",zone_name="hello"} 1 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Discard",volume_name="",volume_type="",zone_name="hello"} 2 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Read",volume_name="",volume_type="",zone_name="hello"} 3 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Sync",volume_name="",volume_type="",zone_name="hello"} 4 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Total",volume_name="",volume_type="",zone_name="hello"} 5 1395066363000 +container_blkio_device_usage_total{container_env_foo_env="prod",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",major="8",minor="0",name="testcontaineralias",operation="Write",volume_name="",volume_type="",zone_name="hello"} 6 1395066363000 +# HELP container_cpu_cfs_burst_periods_total Number of periods when burst occurs. +# TYPE container_cpu_cfs_burst_periods_total counter +container_cpu_cfs_burst_periods_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 1395066363000 +# HELP container_cpu_cfs_burst_seconds_total Total time duration the container has been bursted. +# TYPE container_cpu_cfs_burst_seconds_total counter +container_cpu_cfs_burst_seconds_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.5 1395066363000 # HELP container_cpu_cfs_periods_total Number of elapsed enforcement period intervals. # TYPE container_cpu_cfs_periods_total counter container_cpu_cfs_periods_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 723 1395066363000 @@ -18,12 +24,6 @@ container_cpu_cfs_throttled_periods_total{container_env_foo_env="prod",id="testc # HELP container_cpu_cfs_throttled_seconds_total Total time duration the container has been throttled. # TYPE container_cpu_cfs_throttled_seconds_total counter container_cpu_cfs_throttled_seconds_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.724314 1395066363000 -# HELP container_cpu_cfs_burst_periods_total Number of periods when burst occurs. -# TYPE container_cpu_cfs_burst_periods_total counter -container_cpu_cfs_burst_periods_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 1395066363000 -# HELP container_cpu_cfs_burst_seconds_total Total time duration the container has been bursted. -# TYPE container_cpu_cfs_burst_seconds_total counter -container_cpu_cfs_burst_seconds_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.5 1395066363000 # HELP container_cpu_load_average_10s Value of container cpu load average over the last 10 seconds. # TYPE container_cpu_load_average_10s gauge container_cpu_load_average_10s{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2 1395066363000 @@ -66,82 +66,82 @@ container_custom_app_metric_3{app_test_label="test_value",container_env_foo_env= container_file_descriptors{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000 # HELP container_fs_inodes_free Number of available Inodes # TYPE container_fs_inodes_free gauge -container_fs_inodes_free{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 524288 1395066363000 -container_fs_inodes_free{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 262144 1395066363000 +container_fs_inodes_free{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 524288 1395066363000 +container_fs_inodes_free{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 262144 1395066363000 # HELP container_fs_inodes_total Number of Inodes # TYPE container_fs_inodes_total gauge -container_fs_inodes_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.097152e+06 1395066363000 -container_fs_inodes_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.097152e+06 1395066363000 +container_fs_inodes_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 2.097152e+06 1395066363000 +container_fs_inodes_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 2.097152e+06 1395066363000 +# HELP container_fs_io_cost_indebt_seconds_total Cumulative IOCost debt in seconds +# TYPE container_fs_io_cost_indebt_seconds_total counter +container_fs_io_cost_indebt_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 0.5 1395066363000 +# HELP container_fs_io_cost_indelay_seconds_total Cumulative IOCost delay in seconds +# TYPE container_fs_io_cost_indelay_seconds_total counter +container_fs_io_cost_indelay_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 0.75 1395066363000 +# HELP container_fs_io_cost_usage_seconds_total Cumulative IOCost usage in seconds +# TYPE container_fs_io_cost_usage_seconds_total counter +container_fs_io_cost_usage_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 1.5 1395066363000 +# HELP container_fs_io_cost_wait_seconds_total Cumulative IOCost wait in seconds +# TYPE container_fs_io_cost_wait_seconds_total counter +container_fs_io_cost_wait_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 2.5 1395066363000 # HELP container_fs_io_current Number of I/Os currently in progress # TYPE container_fs_io_current gauge -container_fs_io_current{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 1395066363000 -container_fs_io_current{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 47 1395066363000 +container_fs_io_current{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 42 1395066363000 +container_fs_io_current{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 47 1395066363000 # HELP container_fs_io_time_seconds_total Cumulative count of seconds spent doing I/Os # TYPE container_fs_io_time_seconds_total counter -container_fs_io_time_seconds_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.3e-08 1395066363000 -container_fs_io_time_seconds_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.8e-08 1395066363000 +container_fs_io_time_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.3e-08 1395066363000 +container_fs_io_time_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.8e-08 1395066363000 # HELP container_fs_io_time_weighted_seconds_total Cumulative weighted I/O time in seconds # TYPE container_fs_io_time_weighted_seconds_total counter -container_fs_io_time_weighted_seconds_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.4e-08 1395066363000 -container_fs_io_time_weighted_seconds_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.9e-08 1395066363000 -# HELP container_fs_io_cost_usage_seconds_total Cumulative IOCost usage in seconds -# TYPE container_fs_io_cost_usage_seconds_total counter -container_fs_io_cost_usage_seconds_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.5 1395066363000 -# HELP container_fs_io_cost_wait_seconds_total Cumulative IOCost wait in seconds -# TYPE container_fs_io_cost_wait_seconds_total counter -container_fs_io_cost_wait_seconds_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.5 1395066363000 -# HELP container_fs_io_cost_indebt_seconds_total Cumulative IOCost debt in seconds -# TYPE container_fs_io_cost_indebt_seconds_total counter -container_fs_io_cost_indebt_seconds_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.5 1395066363000 -# HELP container_fs_io_cost_indelay_seconds_total Cumulative IOCost delay in seconds -# TYPE container_fs_io_cost_indelay_seconds_total counter -container_fs_io_cost_indelay_seconds_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.75 1395066363000 +container_fs_io_time_weighted_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.4e-08 1395066363000 +container_fs_io_time_weighted_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.9e-08 1395066363000 # HELP container_fs_limit_bytes Number of bytes that can be consumed by the container on this filesystem. # TYPE container_fs_limit_bytes gauge -container_fs_limit_bytes{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 22 1395066363000 -container_fs_limit_bytes{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 37 1395066363000 +container_fs_limit_bytes{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 22 1395066363000 +container_fs_limit_bytes{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 37 1395066363000 # HELP container_fs_read_seconds_total Cumulative count of seconds spent reading # TYPE container_fs_read_seconds_total counter -container_fs_read_seconds_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.7e-08 1395066363000 -container_fs_read_seconds_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.2e-08 1395066363000 +container_fs_read_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 2.7e-08 1395066363000 +container_fs_read_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.2e-08 1395066363000 # HELP container_fs_reads_bytes_total Cumulative count of bytes read # TYPE container_fs_reads_bytes_total counter -container_fs_reads_bytes_total{container_env_foo_env="prod",device="/dev/sdb",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3 1395066363000 +container_fs_reads_bytes_total{container_env_foo_env="prod",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 3 1395066363000 # HELP container_fs_reads_merged_total Cumulative count of reads merged # TYPE container_fs_reads_merged_total counter -container_fs_reads_merged_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 1395066363000 -container_fs_reads_merged_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 1395066363000 +container_fs_reads_merged_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 25 1395066363000 +container_fs_reads_merged_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 40 1395066363000 # HELP container_fs_reads_total Cumulative count of reads completed # TYPE container_fs_reads_total counter -container_fs_reads_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 24 1395066363000 -container_fs_reads_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 1395066363000 +container_fs_reads_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 24 1395066363000 +container_fs_reads_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 39 1395066363000 # HELP container_fs_sector_reads_total Cumulative count of sector reads completed # TYPE container_fs_sector_reads_total counter -container_fs_sector_reads_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 26 1395066363000 -container_fs_sector_reads_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 41 1395066363000 +container_fs_sector_reads_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 26 1395066363000 +container_fs_sector_reads_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 41 1395066363000 # HELP container_fs_sector_writes_total Cumulative count of sector writes completed # TYPE container_fs_sector_writes_total counter -container_fs_sector_writes_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 1395066363000 -container_fs_sector_writes_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 45 1395066363000 +container_fs_sector_writes_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 40 1395066363000 +container_fs_sector_writes_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 45 1395066363000 # HELP container_fs_usage_bytes Number of bytes that are consumed by the container on this filesystem. # TYPE container_fs_usage_bytes gauge -container_fs_usage_bytes{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 23 1395066363000 -container_fs_usage_bytes{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 38 1395066363000 +container_fs_usage_bytes{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 23 1395066363000 +container_fs_usage_bytes{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 38 1395066363000 # HELP container_fs_write_seconds_total Cumulative count of seconds spent writing # TYPE container_fs_write_seconds_total counter -container_fs_write_seconds_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.1e-08 1395066363000 -container_fs_write_seconds_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.6e-08 1395066363000 +container_fs_write_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.1e-08 1395066363000 +container_fs_write_seconds_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 4.6e-08 1395066363000 # HELP container_fs_writes_bytes_total Cumulative count of bytes written # TYPE container_fs_writes_bytes_total counter -container_fs_writes_bytes_total{container_env_foo_env="prod",device="/dev/sdb",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 6 1395066363000 +container_fs_writes_bytes_total{container_env_foo_env="prod",container_mountpath="",device="/dev/sdb",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 6 1395066363000 # HELP container_fs_writes_merged_total Cumulative count of writes merged # TYPE container_fs_writes_merged_total counter -container_fs_writes_merged_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 1395066363000 -container_fs_writes_merged_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 44 1395066363000 +container_fs_writes_merged_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 39 1395066363000 +container_fs_writes_merged_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 44 1395066363000 # HELP container_fs_writes_total Cumulative count of writes completed # TYPE container_fs_writes_total counter -container_fs_writes_total{container_env_foo_env="prod",device="sda1",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 28 1395066363000 -container_fs_writes_total{container_env_foo_env="prod",device="sda2",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 43 1395066363000 +container_fs_writes_total{container_env_foo_env="prod",container_mountpath="",device="sda1",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 28 1395066363000 +container_fs_writes_total{container_env_foo_env="prod",container_mountpath="",device="sda2",host_mountpoint="",id="testcontainer",image="test",name="testcontaineralias",volume_name="",volume_type="",zone_name="hello"} 43 1395066363000 # HELP container_health_state The result of the container's health check # TYPE container_health_state gauge container_health_state{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000 @@ -160,6 +160,18 @@ container_hugetlb_usage_bytes{container_env_foo_env="prod",id="testcontainer",im # HELP container_last_seen Last time a container was seen by the exporter # TYPE container_last_seen gauge container_last_seen{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.395066363e+09 1395066363000 +# HELP container_llc_occupancy_bytes Last level cache usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). +# TYPE container_llc_occupancy_bytes gauge +container_llc_occupancy_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 162626 1395066363000 +container_llc_occupancy_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 213777 1395066363000 +# HELP container_memory_bandwidth_bytes Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). +# TYPE container_memory_bandwidth_bytes gauge +container_memory_bandwidth_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 4.512312e+06 1395066363000 +container_memory_bandwidth_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 2.173713e+06 1395066363000 +# HELP container_memory_bandwidth_local_bytes Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). +# TYPE container_memory_bandwidth_local_bytes gauge +container_memory_bandwidth_local_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 2.390393e+06 1395066363000 +container_memory_bandwidth_local_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 1.231233e+06 1395066363000 # HELP container_memory_cache Number of bytes of page cache memory. # TYPE container_memory_cache gauge container_memory_cache{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 14 1395066363000 @@ -388,26 +400,26 @@ container_network_udp_usage_total{container_env_foo_env="prod",id="testcontainer # HELP container_oom_events_total Count of out of memory events observed for the container # TYPE container_oom_events_total counter container_oom_events_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 1395066363000 -# HELP container_perf_events_total Perf event metric. -# TYPE container_perf_events_total counter -container_perf_events_total{container_env_foo_env="prod",cpu="0",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 123 1395066363000 -container_perf_events_total{container_env_foo_env="prod",cpu="0",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 321 1395066363000 -container_perf_events_total{container_env_foo_env="prod",cpu="1",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 456 1395066363000 -container_perf_events_total{container_env_foo_env="prod",cpu="1",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 789 1395066363000 # HELP container_perf_events_scaling_ratio Perf event metric scaling ratio. # TYPE container_perf_events_scaling_ratio gauge container_perf_events_scaling_ratio{container_env_foo_env="prod",cpu="0",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1 1395066363000 container_perf_events_scaling_ratio{container_env_foo_env="prod",cpu="0",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.66666666666 1395066363000 container_perf_events_scaling_ratio{container_env_foo_env="prod",cpu="1",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.5 1395066363000 container_perf_events_scaling_ratio{container_env_foo_env="prod",cpu="1",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.33333333333 1395066363000 -# HELP container_perf_uncore_events_total Perf uncore event metric. -# TYPE container_perf_uncore_events_total counter -container_perf_uncore_events_total{container_env_foo_env="prod",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="0",zone_name="hello"} 1.231231512e+09 1395066363000 -container_perf_uncore_events_total{container_env_foo_env="prod",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="1",zone_name="hello"} 1.111231331e+09 1395066363000 +# HELP container_perf_events_total Perf event metric. +# TYPE container_perf_events_total counter +container_perf_events_total{container_env_foo_env="prod",cpu="0",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 123 1395066363000 +container_perf_events_total{container_env_foo_env="prod",cpu="0",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 321 1395066363000 +container_perf_events_total{container_env_foo_env="prod",cpu="1",event="instructions",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 456 1395066363000 +container_perf_events_total{container_env_foo_env="prod",cpu="1",event="instructions_retired",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 789 1395066363000 # HELP container_perf_uncore_events_scaling_ratio Perf uncore event metric scaling ratio. # TYPE container_perf_uncore_events_scaling_ratio gauge container_perf_uncore_events_scaling_ratio{container_env_foo_env="prod",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="0",zone_name="hello"} 1 1395066363000 container_perf_uncore_events_scaling_ratio{container_env_foo_env="prod",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="1",zone_name="hello"} 1 1395066363000 +# HELP container_perf_uncore_events_total Perf uncore event metric. +# TYPE container_perf_uncore_events_total counter +container_perf_uncore_events_total{container_env_foo_env="prod",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="0",zone_name="hello"} 1.231231512e+09 1395066363000 +container_perf_uncore_events_total{container_env_foo_env="prod",event="cas_count_read",id="testcontainer",image="test",name="testcontaineralias",pmu="uncore_imc_0",socket="1",zone_name="hello"} 1.111231331e+09 1395066363000 # HELP container_pressure_cpu_stalled_seconds_total Total time duration no tasks in the container could make progress due to CPU congestion. # TYPE container_pressure_cpu_stalled_seconds_total counter container_pressure_cpu_stalled_seconds_total{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0.0001 1395066363000 @@ -469,15 +481,3 @@ container_threads_max{container_env_foo_env="prod",id="testcontainer",image="tes # HELP container_ulimits_soft Soft ulimit values for the container root process. Unlimited if -1, except priority and nice # TYPE container_ulimits_soft gauge container_ulimits_soft{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",ulimit="max_open_files",zone_name="hello"} 16384 1395066363000 -# HELP container_llc_occupancy_bytes Last level cache usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). -# TYPE container_llc_occupancy_bytes gauge -container_llc_occupancy_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 162626 1395066363000 -container_llc_occupancy_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 213777 1395066363000 -# HELP container_memory_bandwidth_bytes Total memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). -# TYPE container_memory_bandwidth_bytes gauge -container_memory_bandwidth_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 4.512312e+06 1395066363000 -container_memory_bandwidth_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 2.173713e+06 1395066363000 -# HELP container_memory_bandwidth_local_bytes Local memory bandwidth usage statistics for container counted with RDT Memory Bandwidth Monitoring (MBM). -# TYPE container_memory_bandwidth_local_bytes gauge -container_memory_bandwidth_local_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="0",zone_name="hello"} 2.390393e+06 1395066363000 -container_memory_bandwidth_local_bytes{container_env_foo_env="prod",id="testcontainer",image="test",name="testcontaineralias",node_id="1",zone_name="hello"} 1.231233e+06 1395066363000