From d1e4c7b803e5e1986fd8f9aa6b67014886d4a5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulik?= Date: Wed, 8 Apr 2020 23:05:35 +0200 Subject: [PATCH] intelrdt: add mbm stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Szulik --- events.go | 3 + libcontainer/intelrdt/intelrdt.go | 34 +++++++- libcontainer/intelrdt/mbm.go | 102 ++++++++++++++++++++++++ libcontainer/intelrdt/mbm_test.go | 128 ++++++++++++++++++++++++++++++ libcontainer/intelrdt/stats.go | 11 +++ types/events.go | 5 ++ 6 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 libcontainer/intelrdt/mbm.go create mode 100644 libcontainer/intelrdt/mbm_test.go diff --git a/events.go b/events.go index fb3f6302..b2d3087a 100644 --- a/events.go +++ b/events.go @@ -161,6 +161,9 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats { s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot s.IntelRdt.MemBwSchema = is.MemBwSchema } + if intelrdt.IsMbmEnabled() { + s.IntelRdt.MBMStats = is.MBMStats + } } s.NetworkInterfaces = ls.Interfaces diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go index d40ae644..6be7f898 100644 --- a/libcontainer/intelrdt/intelrdt.go +++ b/libcontainer/intelrdt/intelrdt.go @@ -55,6 +55,10 @@ import ( * | | |-- cbm_mask * | | |-- min_cbm_bits * | | |-- num_closids + * | |-- L3_MON + * | | |-- max_threshold_occupancy + * | | |-- mon_features + * | | |-- num_rmids * | |-- MB * | |-- bandwidth_gran * | |-- delay_linear @@ -221,6 +225,17 @@ func init() { isMbaEnabled = true } } + + if flagsSet.MBMTotal || flagsSet.MBMLocal { + if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil { + isMbmEnabled = true + } + + enabledMonFeatures, err = getMonFeatures(intelRdtRoot) + if err != nil { + return + } + } } // Return the mount point path of Intel RDT "resource control" filesysem @@ -300,6 +315,10 @@ func isIntelRdtMounted() bool { type cpuInfoFlags struct { CAT bool // Cache Allocation Technology MBA bool // Memory Bandwidth Allocation + + // Memory Bandwidth Monitoring related. + MBMTotal bool + MBMLocal bool } func parseCpuInfoFile(path string) (cpuInfoFlags, error) { @@ -325,6 +344,10 @@ func parseCpuInfoFile(path string) (cpuInfoFlags, error) { infoFlags.CAT = true case "mba": infoFlags.MBA = true + case "cqm_mbm_total": + infoFlags.MBMTotal = true + case "cqm_mbm_local": + infoFlags.MBMLocal = true } } return infoFlags, nil @@ -589,7 +612,8 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) { schemaRootStrings := strings.Split(tmpRootStrings, "\n") // The L3 cache and memory bandwidth schemata in 'container_id' group - tmpStrings, err := getIntelRdtParamString(m.GetPath(), "schemata") + containerPath := m.GetPath() + tmpStrings, err := getIntelRdtParamString(containerPath, "schemata") if err != nil { return nil, err } @@ -641,6 +665,14 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) { } } + if IsMbmEnabled() { + mbmStats, err := getMBMStats(containerPath) + if err != nil { + return stats, err + } + stats.MBMStats = mbmStats + } + return stats, nil } diff --git a/libcontainer/intelrdt/mbm.go b/libcontainer/intelrdt/mbm.go new file mode 100644 index 00000000..c76f5155 --- /dev/null +++ b/libcontainer/intelrdt/mbm.go @@ -0,0 +1,102 @@ +// +build linux + +package intelrdt + +import ( + "bufio" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +var ( + // The flag to indicate if Intel RDT/MBM is enabled + isMbmEnabled bool + + enabledMonFeatures monFeatures +) + +type monFeatures struct { + mbmTotalBytes bool + mbmLocalBytes bool +} + +// Check if Intel RDT/MBM is enabled +func IsMbmEnabled() bool { + return isMbmEnabled +} + +func getMonFeatures(intelRdtRoot string) (monFeatures, error) { + file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features")) + defer file.Close() + if err != nil { + return monFeatures{}, err + } + return parseMonFeatures(file) +} + +func parseMonFeatures(reader io.Reader) (monFeatures, error) { + scanner := bufio.NewScanner(reader) + + monFeatures := monFeatures{} + + for scanner.Scan() { + + switch feature := scanner.Text(); feature { + + case "mbm_total_bytes": + monFeatures.mbmTotalBytes = true + case "mbm_local_bytes": + monFeatures.mbmLocalBytes = true + default: + logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature) + } + } + + return monFeatures, scanner.Err() +} + +func getMBMStats(containerPath string) (*[]MBMNumaNodeStats, error) { + var mbmStats []MBMNumaNodeStats + + numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data")) + if err != nil { + return &mbmStats, err + } + + for _, file := range numaFiles { + if file.IsDir() { + numaStats, err := getMBMNumaNodeStats(filepath.Join(containerPath, "mon_data", file.Name())) + if err != nil { + return &mbmStats, nil + } + mbmStats = append(mbmStats, *numaStats) + } + } + + return &mbmStats, nil +} + +func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) { + stats := &MBMNumaNodeStats{} + if enabledMonFeatures.mbmTotalBytes { + mbmTotalBytes, err := getIntelRdtParamUint(numaPath, "mbm_total_bytes") + if err != nil { + return nil, err + } + stats.MBMTotalBytes = mbmTotalBytes + } + + if enabledMonFeatures.mbmLocalBytes { + mbmLocalBytes, err := getIntelRdtParamUint(numaPath, "mbm_local_bytes") + if err != nil { + return nil, err + } + stats.MBMLocalBytes = mbmLocalBytes + } + + return stats, nil +} diff --git a/libcontainer/intelrdt/mbm_test.go b/libcontainer/intelrdt/mbm_test.go new file mode 100644 index 00000000..9ddf8ba2 --- /dev/null +++ b/libcontainer/intelrdt/mbm_test.go @@ -0,0 +1,128 @@ +// +build linux + +package intelrdt + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestParseMonFeatures(t *testing.T) { + t.Run("All features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures( + strings.NewReader("mbm_total_bytes\nmbm_local_bytes")) + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{true, true} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Cannot gather all features!") + } + }) + + t.Run("No features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures(strings.NewReader("")) + + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{false, false} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Expected no features available but there is any!") + } + }) +} + +func mockMBM(NUMANodes []string, mocks map[string]uint64) (string, error) { + testDir, err := ioutil.TempDir("", "rdt_mbm_test") + if err != nil { + return "", err + } + monDataPath := filepath.Join(testDir, "mon_data") + + for _, numa := range NUMANodes { + numaPath := filepath.Join(monDataPath, numa) + err = os.MkdirAll(numaPath, os.ModePerm) + if err != nil { + return "", err + } + + for fileName, value := range mocks { + err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777) + if err != nil { + return "", err + } + } + + } + + return testDir, nil +} + +func TestGetMbmStats(t *testing.T) { + mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} + + mocksFilesToCreate := map[string]uint64{ + "mbm_total_bytes": 9123911, + "mbm_local_bytes": 2361361, + } + + mockedMBM, err := mockMBM(mocksNUMANodesToCreate, mocksFilesToCreate) + + defer func() { + err := os.RemoveAll(mockedMBM) + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + + t.Run("Gather mbm", func(t *testing.T) { + enabledMonFeatures.mbmTotalBytes = true + enabledMonFeatures.mbmLocalBytes = true + + stats, err := getMBMStats(mockedMBM) + if err != nil { + t.Fatal(err) + } + + if len(*stats) != len(mocksNUMANodesToCreate) { + t.Fatalf("Wrong number of stats slices from NUMA nodes. Expected: %v but got: %v", + len(mocksNUMANodesToCreate), len(*stats)) + } + + checkStatCorrection := func(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) { + if got.MBMTotalBytes != expected.MBMTotalBytes { + t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v", + expected.MBMTotalBytes, + got.MBMTotalBytes) + } + + if got.MBMLocalBytes != expected.MBMLocalBytes { + t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v", + expected.MBMLocalBytes, + got.MBMLocalBytes) + } + + } + + expectedStats := MBMNumaNodeStats{ + MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"], + MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"], + } + + checkStatCorrection((*stats)[0], expectedStats, t) + checkStatCorrection((*stats)[1], expectedStats, t) + }) +} diff --git a/libcontainer/intelrdt/stats.go b/libcontainer/intelrdt/stats.go index df5686f3..f90293be 100644 --- a/libcontainer/intelrdt/stats.go +++ b/libcontainer/intelrdt/stats.go @@ -15,6 +15,14 @@ type MemBwInfo struct { NumClosids uint64 `json:"num_closids,omitempty"` } +type MBMNumaNodeStats struct { + // The 'mbm_total_bytes' in 'container_id' group + MBMTotalBytes uint64 `json:"mbm_total_bytes,omitempty"` + + // The 'mbm_local_bytes' in 'container_id' group + MBMLocalBytes uint64 `json:"mbm_local_bytes,omitempty"` +} + type Stats struct { // The read-only L3 cache information L3CacheInfo *L3CacheInfo `json:"l3_cache_info,omitempty"` @@ -33,6 +41,9 @@ type Stats struct { // The memory bandwidth schema in 'container_id' group MemBwSchema string `json:"mem_bw_schema,omitempty"` + + // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group + MBMStats *[]MBMNumaNodeStats `json:"mbm_statistics,omitempty"` } func NewStats() *Stats { diff --git a/types/events.go b/types/events.go index c6f0e97a..bdc0a515 100644 --- a/types/events.go +++ b/types/events.go @@ -1,5 +1,7 @@ package types +import "github.com/opencontainers/runc/libcontainer/intelrdt" + // Event struct for encoding the event data to json. type Event struct { Type string `json:"type"` @@ -113,6 +115,9 @@ type IntelRdt struct { // The memory bandwidth schema in 'container_id' group MemBwSchema string `json:"mem_bw_schema,omitempty"` + + // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group + MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_statistics,omitempty"` } type NetworkInterface struct {