diff --git a/events.go b/events.go index f0f74ef0..0a687c32 100644 --- a/events.go +++ b/events.go @@ -163,6 +163,12 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats { s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot s.IntelRdt.MemBwSchema = is.MemBwSchema } + if intelrdt.IsMBMEnabled() { + s.IntelRdt.MBMStats = is.MBMStats + } + if intelrdt.IsCMTEnabled() { + s.IntelRdt.CMTStats = is.CMTStats + } } s.NetworkInterfaces = ls.Interfaces diff --git a/libcontainer/intelrdt/cmt.go b/libcontainer/intelrdt/cmt.go new file mode 100644 index 00000000..5c406e10 --- /dev/null +++ b/libcontainer/intelrdt/cmt.go @@ -0,0 +1,22 @@ +package intelrdt + +var ( + cmtEnabled bool +) + +// Check if Intel RDT/CMT is enabled. +func IsCMTEnabled() bool { + return cmtEnabled +} + +func getCMTNumaNodeStats(numaPath string) (*CMTNumaNodeStats, error) { + stats := &CMTNumaNodeStats{} + + llcOccupancy, err := getIntelRdtParamUint(numaPath, "llc_occupancy") + if err != nil { + return nil, err + } + stats.LLCOccupancy = llcOccupancy + + return stats, nil +} diff --git a/libcontainer/intelrdt/cmt_test.go b/libcontainer/intelrdt/cmt_test.go new file mode 100644 index 00000000..e061695e --- /dev/null +++ b/libcontainer/intelrdt/cmt_test.go @@ -0,0 +1,56 @@ +package intelrdt + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGetCMTNumaNodeStats(t *testing.T) { + mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} + + mocksFilesToCreate := map[string]uint64{ + "llc_occupancy": 9123911, + } + + mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate) + + defer func() { + err := os.RemoveAll(mockedL3_MON) + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + + t.Run("Gather mbm", func(t *testing.T) { + enabledMonFeatures.llcOccupancy = true + + stats := make([]CMTNumaNodeStats, 0, len(mocksNUMANodesToCreate)) + for _, numa := range mocksNUMANodesToCreate { + other, err := getCMTNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa)) + if err != nil { + t.Fatal(err) + } + stats = append(stats, *other) + } + + expectedStats := CMTNumaNodeStats{ + LLCOccupancy: mocksFilesToCreate["llc_occupancy"], + } + + checkCMTStatCorrection(stats[0], expectedStats, t) + checkCMTStatCorrection(stats[1], expectedStats, t) + }) +} + +func checkCMTStatCorrection(got CMTNumaNodeStats, expected CMTNumaNodeStats, t *testing.T) { + if got.LLCOccupancy != expected.LLCOccupancy { + t.Fatalf("Wrong value of `llc_occupancy`. Expected: %v but got: %v", + expected.LLCOccupancy, + got.LLCOccupancy) + } +} diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go index b19580f1..5b19d55a 100644 --- a/libcontainer/intelrdt/intelrdt.go +++ b/libcontainer/intelrdt/intelrdt.go @@ -55,6 +55,10 @@ import ( * | | |-- cbm_mask * | | |-- min_cbm_bits * | | |-- num_closids + * | |-- L3_MON + * | | |-- max_threshold_occupancy + * | | |-- mon_features + * | | |-- num_rmids * | |-- MB * | |-- bandwidth_gran * | |-- delay_linear @@ -191,8 +195,7 @@ type intelRdtData struct { // Check if Intel RDT sub-features are enabled in init() func init() { // 1. Check if hardware and kernel support Intel RDT sub-features - // "cat_l3" flag for CAT and "mba" flag for MBA - isCatFlagSet, isMbaFlagSet, err := parseCpuInfoFile("/proc/cpuinfo") + flagsSet, err := parseCpuInfoFile("/proc/cpuinfo") if err != nil { return } @@ -207,7 +210,7 @@ func init() { // "resource control" filesystem. Intel RDT sub-features can be // selectively disabled or enabled by kernel command line // (e.g., rdt=!l3cat,mba) in 4.14 and newer kernel - if isCatFlagSet { + if flagsSet.CAT { if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3")); err == nil { isCatEnabled = true } @@ -217,11 +220,23 @@ func init() { // MBA should be enabled because MBA Software Controller // depends on MBA isMbaEnabled = true - } else if isMbaFlagSet { + } else if flagsSet.MBA { if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "MB")); err == nil { isMbaEnabled = true } } + + if flagsSet.MBMTotal || flagsSet.MBMLocal { + if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil { + mbmEnabled = true + cmtEnabled = true + } + + enabledMonFeatures, err = getMonFeatures(intelRdtRoot) + if err != nil { + return + } + } } // Return the mount point path of Intel RDT "resource control" filesysem @@ -298,13 +313,21 @@ func isIntelRdtMounted() bool { return true } -func parseCpuInfoFile(path string) (bool, bool, error) { - isCatFlagSet := false - isMbaFlagSet := false +type cpuInfoFlags struct { + CAT bool // Cache Allocation Technology + MBA bool // Memory Bandwidth Allocation + + // Memory Bandwidth Monitoring related. + MBMTotal bool + MBMLocal bool +} + +func parseCpuInfoFile(path string) (cpuInfoFlags, error) { + infoFlags := cpuInfoFlags{} f, err := os.Open(path) if err != nil { - return false, false, err + return infoFlags, err } defer f.Close() @@ -319,19 +342,23 @@ func parseCpuInfoFile(path string) (bool, bool, error) { for _, flag := range flags { switch flag { case "cat_l3": - isCatFlagSet = true + infoFlags.CAT = true case "mba": - isMbaFlagSet = true + infoFlags.MBA = true + case "cqm_mbm_total": + infoFlags.MBMTotal = true + case "cqm_mbm_local": + infoFlags.MBMLocal = true } } - return isCatFlagSet, isMbaFlagSet, nil + return infoFlags, nil } } if err := s.Err(); err != nil { - return false, false, err + return infoFlags, err } - return isCatFlagSet, isMbaFlagSet, nil + return infoFlags, nil } func parseUint(s string, base, bitSize int) (uint64, error) { @@ -586,7 +613,8 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) { schemaRootStrings := strings.Split(tmpRootStrings, "\n") // The L3 cache and memory bandwidth schemata in 'container_id' group - tmpStrings, err := getIntelRdtParamString(m.GetPath(), "schemata") + containerPath := m.GetPath() + tmpStrings, err := getIntelRdtParamString(containerPath, "schemata") if err != nil { return nil, err } @@ -638,6 +666,11 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) { } } + err = getMonitoringStats(containerPath, stats) + if err != nil { + return nil, err + } + return stats, nil } diff --git a/libcontainer/intelrdt/mbm.go b/libcontainer/intelrdt/mbm.go new file mode 100644 index 00000000..3730bab5 --- /dev/null +++ b/libcontainer/intelrdt/mbm.go @@ -0,0 +1,34 @@ +// +build linux + +package intelrdt + +var ( + // The flag to indicate if Intel RDT/MBM is enabled + mbmEnabled bool +) + +// Check if Intel RDT/MBM is enabled. +func IsMBMEnabled() bool { + return mbmEnabled +} + +func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) { + stats := &MBMNumaNodeStats{} + if enabledMonFeatures.mbmTotalBytes { + mbmTotalBytes, err := getIntelRdtParamUint(numaPath, "mbm_total_bytes") + if err != nil { + return nil, err + } + stats.MBMTotalBytes = mbmTotalBytes + } + + if enabledMonFeatures.mbmLocalBytes { + mbmLocalBytes, err := getIntelRdtParamUint(numaPath, "mbm_local_bytes") + if err != nil { + return nil, err + } + stats.MBMLocalBytes = mbmLocalBytes + } + + return stats, nil +} diff --git a/libcontainer/intelrdt/mbm_test.go b/libcontainer/intelrdt/mbm_test.go new file mode 100644 index 00000000..ae0f202c --- /dev/null +++ b/libcontainer/intelrdt/mbm_test.go @@ -0,0 +1,68 @@ +// +build linux + +package intelrdt + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGetMBMNumaNodeStats(t *testing.T) { + mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} + + mocksFilesToCreate := map[string]uint64{ + "mbm_total_bytes": 9123911, + "mbm_local_bytes": 2361361, + } + + mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate) + + defer func() { + err := os.RemoveAll(mockedL3_MON) + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + + t.Run("Gather mbm", func(t *testing.T) { + enabledMonFeatures.mbmTotalBytes = true + enabledMonFeatures.mbmLocalBytes = true + + stats := make([]MBMNumaNodeStats, 0, len(mocksNUMANodesToCreate)) + for _, numa := range mocksNUMANodesToCreate { + other, err := getMBMNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa)) + if err != nil { + t.Fatal(err) + } + stats = append(stats, *other) + } + + expectedStats := MBMNumaNodeStats{ + MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"], + MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"], + } + + checkMBMStatCorrection(stats[0], expectedStats, t) + checkMBMStatCorrection(stats[1], expectedStats, t) + }) +} + +func checkMBMStatCorrection(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) { + if got.MBMTotalBytes != expected.MBMTotalBytes { + t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v", + expected.MBMTotalBytes, + got.MBMTotalBytes) + } + + if got.MBMLocalBytes != expected.MBMLocalBytes { + t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v", + expected.MBMLocalBytes, + got.MBMLocalBytes) + } + +} diff --git a/libcontainer/intelrdt/monitoring.go b/libcontainer/intelrdt/monitoring.go new file mode 100644 index 00000000..4ccc8eae --- /dev/null +++ b/libcontainer/intelrdt/monitoring.go @@ -0,0 +1,85 @@ +package intelrdt + +import ( + "bufio" + "github.com/sirupsen/logrus" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +var ( + enabledMonFeatures monFeatures +) + +type monFeatures struct { + mbmTotalBytes bool + mbmLocalBytes bool + llcOccupancy bool +} + +func getMonFeatures(intelRdtRoot string) (monFeatures, error) { + file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features")) + defer file.Close() + if err != nil { + return monFeatures{}, err + } + return parseMonFeatures(file) +} + +func parseMonFeatures(reader io.Reader) (monFeatures, error) { + scanner := bufio.NewScanner(reader) + + monFeatures := monFeatures{} + + for scanner.Scan() { + switch feature := scanner.Text(); feature { + case "mbm_total_bytes": + monFeatures.mbmTotalBytes = true + case "mbm_local_bytes": + monFeatures.mbmLocalBytes = true + case "llc_occupancy": + monFeatures.llcOccupancy = true + default: + logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature) + } + } + + return monFeatures, scanner.Err() +} + +func getMonitoringStats(containerPath string, stats *Stats) error { + numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data")) + if err != nil { + return err + } + + var mbmStats []MBMNumaNodeStats + var cmtStats []CMTNumaNodeStats + + for _, file := range numaFiles { + if file.IsDir() { + numaPath := filepath.Join(containerPath, "mon_data", file.Name()) + if IsMBMEnabled() { + numaMBMStats, err := getMBMNumaNodeStats(numaPath) + if err != nil { + return err + } + mbmStats = append(mbmStats, *numaMBMStats) + } + if IsCMTEnabled() { + numaCMTStats, err := getCMTNumaNodeStats(numaPath) + if err != nil { + return err + } + cmtStats = append(cmtStats, *numaCMTStats) + } + } + } + + stats.MBMStats = &mbmStats + stats.CMTStats = &cmtStats + + return err +} diff --git a/libcontainer/intelrdt/monitoring_test.go b/libcontainer/intelrdt/monitoring_test.go new file mode 100644 index 00000000..ec886a7e --- /dev/null +++ b/libcontainer/intelrdt/monitoring_test.go @@ -0,0 +1,118 @@ +package intelrdt + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestParseMonFeatures(t *testing.T) { + t.Run("All features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures( + strings.NewReader("mbm_total_bytes\nmbm_local_bytes\nllc_occupancy")) + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{true, true, true} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Cannot gather all features!") + } + }) + + t.Run("No features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures(strings.NewReader("")) + + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{false, false, false} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Expected no features available but there is any!") + } + }) +} + +func mockResctrlL3_MON(NUMANodes []string, mocks map[string]uint64) (string, error) { + testDir, err := ioutil.TempDir("", "rdt_mbm_test") + if err != nil { + return "", err + } + monDataPath := filepath.Join(testDir, "mon_data") + + for _, numa := range NUMANodes { + numaPath := filepath.Join(monDataPath, numa) + err = os.MkdirAll(numaPath, os.ModePerm) + if err != nil { + return "", err + } + + for fileName, value := range mocks { + err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777) + if err != nil { + return "", err + } + } + + } + + return testDir, nil +} + +func TestGetMonitoringStats(t *testing.T) { + enabledMonFeatures.mbmTotalBytes = true + enabledMonFeatures.mbmLocalBytes = true + enabledMonFeatures.llcOccupancy = true + mbmEnabled = true + cmtEnabled = true + + mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} + + mocksFilesToCreate := map[string]uint64{ + "mbm_total_bytes": 9123911, + "mbm_local_bytes": 2361361, + "llc_occupancy": 123331, + } + + mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate) + + defer func() { + err := os.RemoveAll(mockedL3_MON) + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + + t.Run("Gather monitoring stats", func(t *testing.T) { + var stats Stats + err := getMonitoringStats(mockedL3_MON, &stats) + if err != nil { + t.Fatal(err) + } + + expectedMBMStats := MBMNumaNodeStats{ + MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"], + MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"], + } + + expectedCMTStats := CMTNumaNodeStats{LLCOccupancy: mocksFilesToCreate["llc_occupancy"]} + + for _, gotMBMStat := range *stats.MBMStats { + checkMBMStatCorrection(gotMBMStat, expectedMBMStats, t) + } + + for _, gotCMTStat := range *stats.CMTStats { + checkCMTStatCorrection(gotCMTStat, expectedCMTStats, t) + } + }) +} diff --git a/libcontainer/intelrdt/stats.go b/libcontainer/intelrdt/stats.go index df5686f3..eeb0ee9f 100644 --- a/libcontainer/intelrdt/stats.go +++ b/libcontainer/intelrdt/stats.go @@ -15,6 +15,19 @@ type MemBwInfo struct { NumClosids uint64 `json:"num_closids,omitempty"` } +type MBMNumaNodeStats struct { + // The 'mbm_total_bytes' in 'container_id' group. + MBMTotalBytes uint64 `json:"mbm_total_bytes,omitempty"` + + // The 'mbm_local_bytes' in 'container_id' group. + MBMLocalBytes uint64 `json:"mbm_local_bytes,omitempty"` +} + +type CMTNumaNodeStats struct { + // The 'llc_occupancy' in 'container_id' group. + LLCOccupancy uint64 `json:"llc_occupancy,omitempty"` +} + type Stats struct { // The read-only L3 cache information L3CacheInfo *L3CacheInfo `json:"l3_cache_info,omitempty"` @@ -33,6 +46,12 @@ type Stats struct { // The memory bandwidth schema in 'container_id' group MemBwSchema string `json:"mem_bw_schema,omitempty"` + + // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group + MBMStats *[]MBMNumaNodeStats `json:"mbm_stats,omitempty"` + + // The cache monitoring technology statistics from NUMA nodes in 'container_id' group + CMTStats *[]CMTNumaNodeStats `json:"cmt_stats,omitempty"` } func NewStats() *Stats { diff --git a/types/events.go b/types/events.go index 366c0a34..6f9a12f1 100644 --- a/types/events.go +++ b/types/events.go @@ -1,5 +1,7 @@ package types +import "github.com/opencontainers/runc/libcontainer/intelrdt" + // Event struct for encoding the event data to json. type Event struct { Type string `json:"type"` @@ -115,6 +117,12 @@ type IntelRdt struct { // The memory bandwidth schema in 'container_id' group MemBwSchema string `json:"mem_bw_schema,omitempty"` + + // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group + MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_stats,omitempty"` + + // The cache monitoring technology statistics from NUMA nodes in 'container_id' group + CMTStats *[]intelrdt.CMTNumaNodeStats `json:"cmt_stats,omitempty"` } type NetworkInterface struct {