Exposing memory.numa_stats
Making information on page usage by type and NUMA node available Signed-off-by: Maciej "Iwan" Iwanowski <maciej.iwanowski@intel.com>
This commit is contained in:
iwankgb
Maciej "Iwan" Iwanowski
parent
4a9e1747da
commit
7fe0a98e79
|
|
@ -5,7 +5,9 @@ package fs
|
|||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
|
@ -16,8 +18,16 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
cgroupMemorySwapLimit = "memory.memsw.limit_in_bytes"
|
||||
cgroupMemoryLimit = "memory.limit_in_bytes"
|
||||
numaNodeSymbol = "N"
|
||||
numaStatColumnSeparator = " "
|
||||
numaStatKeyValueSeparator = "="
|
||||
numaStatMaxColumns = math.MaxUint8 + 1
|
||||
numaStatValueIndex = 1
|
||||
numaStatTypeIndex = 0
|
||||
numaStatColumnSliceLength = 2
|
||||
cgroupMemorySwapLimit = "memory.memsw.limit_in_bytes"
|
||||
cgroupMemoryLimit = "memory.limit_in_bytes"
|
||||
cgroupMemoryPagesByNuma = "memory.numa_stat"
|
||||
)
|
||||
|
||||
type MemoryGroup struct {
|
||||
|
|
@ -209,6 +219,13 @@ func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
|
|||
if value == 1 {
|
||||
stats.MemoryStats.UseHierarchy = true
|
||||
}
|
||||
|
||||
pagesByNUMA, err := getPageUsageByNUMA(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stats.MemoryStats.PageUsageByNUMA = pagesByNUMA
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -269,3 +286,77 @@ func getMemoryData(path, name string) (cgroups.MemoryData, error) {
|
|||
|
||||
return memoryData, nil
|
||||
}
|
||||
|
||||
func getPageUsageByNUMA(cgroupPath string) (cgroups.PageUsageByNUMA, error) {
|
||||
stats := cgroups.PageUsageByNUMA{}
|
||||
|
||||
file, err := os.Open(path.Join(cgroupPath, cgroupMemoryPagesByNuma))
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
var statsType string
|
||||
statsByType := cgroups.PageStats{Nodes: map[uint8]uint64{}}
|
||||
columns := strings.SplitN(scanner.Text(), numaStatColumnSeparator, numaStatMaxColumns)
|
||||
|
||||
for _, column := range columns {
|
||||
pagesByNode := strings.SplitN(column, numaStatKeyValueSeparator, numaStatColumnSliceLength)
|
||||
|
||||
if strings.HasPrefix(pagesByNode[numaStatTypeIndex], numaNodeSymbol) {
|
||||
nodeID, err := strconv.ParseUint(pagesByNode[numaStatTypeIndex][1:], 10, 8)
|
||||
if err != nil {
|
||||
return cgroups.PageUsageByNUMA{}, err
|
||||
}
|
||||
|
||||
statsByType.Nodes[uint8(nodeID)], err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
|
||||
if err != nil {
|
||||
return cgroups.PageUsageByNUMA{}, err
|
||||
}
|
||||
} else {
|
||||
statsByType.Total, err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
|
||||
if err != nil {
|
||||
return cgroups.PageUsageByNUMA{}, err
|
||||
}
|
||||
|
||||
statsType = pagesByNode[numaStatTypeIndex]
|
||||
}
|
||||
|
||||
err := addNUMAStatsByType(&stats, statsByType, statsType)
|
||||
if err != nil {
|
||||
return cgroups.PageUsageByNUMA{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
err = scanner.Err()
|
||||
if err != nil {
|
||||
return cgroups.PageUsageByNUMA{}, err
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func addNUMAStatsByType(stats *cgroups.PageUsageByNUMA, byTypeStats cgroups.PageStats, statsType string) error {
|
||||
switch statsType {
|
||||
case "total":
|
||||
stats.Total = byTypeStats
|
||||
case "file":
|
||||
stats.File = byTypeStats
|
||||
case "anon":
|
||||
stats.Anon = byTypeStats
|
||||
case "unevictable":
|
||||
stats.Unevictable = byTypeStats
|
||||
case "hierarchical_total":
|
||||
stats.Hierarchical.Total = byTypeStats
|
||||
case "hierarchical_file":
|
||||
stats.Hierarchical.File = byTypeStats
|
||||
case "hierarchical_anon":
|
||||
stats.Hierarchical.Anon = byTypeStats
|
||||
case "hierarchical_unevictable":
|
||||
stats.Hierarchical.Unevictable = byTypeStats
|
||||
default:
|
||||
return fmt.Errorf("unsupported NUMA page type found: %s", statsType)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,18 @@ rss 1024`
|
|||
memoryFailcnt = "100\n"
|
||||
memoryLimitContents = "8192\n"
|
||||
memoryUseHierarchyContents = "1\n"
|
||||
memoryNUMAStatContents = `total=44611 N0=32631 N1=7501 N2=1982 N3=2497
|
||||
file=44428 N0=32614 N1=7335 N2=1982 N3=2497
|
||||
anon=183 N0=17 N1=166 N2=0 N3=0
|
||||
unevictable=0 N0=0 N1=0 N2=0 N3=0
|
||||
hierarchical_total=768133 N0=509113 N1=138887 N2=20464 N3=99669
|
||||
hierarchical_file=722017 N0=496516 N1=119997 N2=20181 N3=85323
|
||||
hierarchical_anon=46096 N0=12597 N1=18890 N2=283 N3=14326
|
||||
hierarchical_unevictable=20 N0=0 N1=0 N2=0 N3=20`
|
||||
memoryNUMAStatNoHierarchyContents = `total=44611 N0=32631 N1=7501 N2=1982 N3=2497
|
||||
file=44428 N0=32614 N1=7335 N2=1982 N3=2497
|
||||
anon=183 N0=17 N1=166 N2=0 N3=0
|
||||
unevictable=0 N0=0 N1=0 N2=0 N3=0`
|
||||
)
|
||||
|
||||
func TestMemorySetMemory(t *testing.T) {
|
||||
|
|
@ -276,6 +288,7 @@ func TestMemoryStats(t *testing.T) {
|
|||
"memory.kmem.failcnt": memoryFailcnt,
|
||||
"memory.kmem.limit_in_bytes": memoryLimitContents,
|
||||
"memory.use_hierarchy": memoryUseHierarchyContents,
|
||||
"memory.numa_stat": memoryNUMAStatContents,
|
||||
})
|
||||
|
||||
memory := &MemoryGroup{}
|
||||
|
|
@ -284,7 +297,21 @@ func TestMemoryStats(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectedStats := cgroups.MemoryStats{Cache: 512, Usage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, SwapUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, KernelUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, Stats: map[string]uint64{"cache": 512, "rss": 1024}, UseHierarchy: true}
|
||||
expectedStats := cgroups.MemoryStats{Cache: 512, Usage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, SwapUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, KernelUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, Stats: map[string]uint64{"cache": 512, "rss": 1024}, UseHierarchy: true,
|
||||
PageUsageByNUMA: cgroups.PageUsageByNUMA{
|
||||
PageUsageByNUMAInner: cgroups.PageUsageByNUMAInner{
|
||||
Total: cgroups.PageStats{Total: 44611, Nodes: map[uint8]uint64{0: 32631, 1: 7501, 2: 1982, 3: 2497}},
|
||||
File: cgroups.PageStats{Total: 44428, Nodes: map[uint8]uint64{0: 32614, 1: 7335, 2: 1982, 3: 2497}},
|
||||
Anon: cgroups.PageStats{Total: 183, Nodes: map[uint8]uint64{0: 17, 1: 166, 2: 0, 3: 0}},
|
||||
Unevictable: cgroups.PageStats{Total: 0, Nodes: map[uint8]uint64{0: 0, 1: 0, 2: 0, 3: 0}},
|
||||
},
|
||||
Hierarchical: cgroups.PageUsageByNUMAInner{
|
||||
Total: cgroups.PageStats{Total: 768133, Nodes: map[uint8]uint64{0: 509113, 1: 138887, 2: 20464, 3: 99669}},
|
||||
File: cgroups.PageStats{Total: 722017, Nodes: map[uint8]uint64{0: 496516, 1: 119997, 2: 20181, 3: 85323}},
|
||||
Anon: cgroups.PageStats{Total: 46096, Nodes: map[uint8]uint64{0: 12597, 1: 18890, 2: 283, 3: 14326}},
|
||||
Unevictable: cgroups.PageStats{Total: 20, Nodes: map[uint8]uint64{0: 0, 1: 0, 2: 0, 3: 20}},
|
||||
},
|
||||
}}
|
||||
expectMemoryStatEquals(t, expectedStats, actualStats.MemoryStats)
|
||||
}
|
||||
|
||||
|
|
@ -454,3 +481,26 @@ func TestMemorySetOomControl(t *testing.T) {
|
|||
t.Fatalf("Got the wrong value, set memory.oom_control failed.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoHierarchicalNumaStat(t *testing.T) {
|
||||
helper := NewCgroupTestUtil("memory", t)
|
||||
defer helper.cleanup()
|
||||
helper.writeFileContents(map[string]string{
|
||||
"memory.numa_stat": memoryNUMAStatNoHierarchyContents,
|
||||
})
|
||||
|
||||
actualStats, err := getPageUsageByNUMA(helper.CgroupPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pageUsageByNUMA := cgroups.PageUsageByNUMA{
|
||||
PageUsageByNUMAInner: cgroups.PageUsageByNUMAInner{
|
||||
Total: cgroups.PageStats{Total: 44611, Nodes: map[uint8]uint64{0: 32631, 1: 7501, 2: 1982, 3: 2497}},
|
||||
File: cgroups.PageStats{Total: 44428, Nodes: map[uint8]uint64{0: 32614, 1: 7335, 2: 1982, 3: 2497}},
|
||||
Anon: cgroups.PageStats{Total: 183, Nodes: map[uint8]uint64{0: 17, 1: 166, 2: 0, 3: 0}},
|
||||
Unevictable: cgroups.PageStats{Total: 0, Nodes: map[uint8]uint64{0: 0, 1: 0, 2: 0, 3: 0}},
|
||||
},
|
||||
Hierarchical: cgroups.PageUsageByNUMAInner{},
|
||||
}
|
||||
expectPageUsageByNUMAEquals(t, pageUsageByNUMA, actualStats)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,11 +4,10 @@ package fs
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/opencontainers/runc/libcontainer/cgroups"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func blkioStatEntryEquals(expected, actual []cgroups.BlkioStatEntry) error {
|
||||
|
|
@ -26,57 +25,47 @@ func blkioStatEntryEquals(expected, actual []cgroups.BlkioStatEntry) error {
|
|||
|
||||
func expectBlkioStatsEquals(t *testing.T, expected, actual cgroups.BlkioStats) {
|
||||
if err := blkioStatEntryEquals(expected.IoServiceBytesRecursive, actual.IoServiceBytesRecursive); err != nil {
|
||||
logrus.Printf("blkio IoServiceBytesRecursive do not match - %s\n", err)
|
||||
t.Fail()
|
||||
t.Errorf("blkio IoServiceBytesRecursive do not match - %s\n", err)
|
||||
}
|
||||
|
||||
if err := blkioStatEntryEquals(expected.IoServicedRecursive, actual.IoServicedRecursive); err != nil {
|
||||
logrus.Printf("blkio IoServicedRecursive do not match - %s\n", err)
|
||||
t.Fail()
|
||||
t.Errorf("blkio IoServicedRecursive do not match - %s\n", err)
|
||||
}
|
||||
|
||||
if err := blkioStatEntryEquals(expected.IoQueuedRecursive, actual.IoQueuedRecursive); err != nil {
|
||||
logrus.Printf("blkio IoQueuedRecursive do not match - %s\n", err)
|
||||
t.Fail()
|
||||
t.Errorf("blkio IoQueuedRecursive do not match - %s\n", err)
|
||||
}
|
||||
|
||||
if err := blkioStatEntryEquals(expected.SectorsRecursive, actual.SectorsRecursive); err != nil {
|
||||
logrus.Printf("blkio SectorsRecursive do not match - %s\n", err)
|
||||
t.Fail()
|
||||
t.Errorf("blkio SectorsRecursive do not match - %s\n", err)
|
||||
}
|
||||
|
||||
if err := blkioStatEntryEquals(expected.IoServiceTimeRecursive, actual.IoServiceTimeRecursive); err != nil {
|
||||
logrus.Printf("blkio IoServiceTimeRecursive do not match - %s\n", err)
|
||||
t.Fail()
|
||||
t.Errorf("blkio IoServiceTimeRecursive do not match - %s\n", err)
|
||||
}
|
||||
|
||||
if err := blkioStatEntryEquals(expected.IoWaitTimeRecursive, actual.IoWaitTimeRecursive); err != nil {
|
||||
logrus.Printf("blkio IoWaitTimeRecursive do not match - %s\n", err)
|
||||
t.Fail()
|
||||
t.Errorf("blkio IoWaitTimeRecursive do not match - %s\n", err)
|
||||
}
|
||||
|
||||
if err := blkioStatEntryEquals(expected.IoMergedRecursive, actual.IoMergedRecursive); err != nil {
|
||||
logrus.Printf("blkio IoMergedRecursive do not match - %v vs %v\n", expected.IoMergedRecursive, actual.IoMergedRecursive)
|
||||
t.Fail()
|
||||
t.Errorf("blkio IoMergedRecursive do not match - %v vs %v\n", expected.IoMergedRecursive, actual.IoMergedRecursive)
|
||||
}
|
||||
|
||||
if err := blkioStatEntryEquals(expected.IoTimeRecursive, actual.IoTimeRecursive); err != nil {
|
||||
logrus.Printf("blkio IoTimeRecursive do not match - %s\n", err)
|
||||
t.Fail()
|
||||
t.Errorf("blkio IoTimeRecursive do not match - %s\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
func expectThrottlingDataEquals(t *testing.T, expected, actual cgroups.ThrottlingData) {
|
||||
if expected != actual {
|
||||
logrus.Printf("Expected throttling data %v but found %v\n", expected, actual)
|
||||
t.Fail()
|
||||
t.Errorf("Expected throttling data %v but found %v\n", expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
func expectHugetlbStatEquals(t *testing.T, expected, actual cgroups.HugetlbStats) {
|
||||
if expected != actual {
|
||||
logrus.Printf("Expected hugetlb stats %v but found %v\n", expected, actual)
|
||||
t.Fail()
|
||||
t.Errorf("Expected hugetlb stats %v but found %v\n", expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -84,40 +73,61 @@ func expectMemoryStatEquals(t *testing.T, expected, actual cgroups.MemoryStats)
|
|||
expectMemoryDataEquals(t, expected.Usage, actual.Usage)
|
||||
expectMemoryDataEquals(t, expected.SwapUsage, actual.SwapUsage)
|
||||
expectMemoryDataEquals(t, expected.KernelUsage, actual.KernelUsage)
|
||||
expectPageUsageByNUMAEquals(t, expected.PageUsageByNUMA, actual.PageUsageByNUMA)
|
||||
|
||||
if expected.UseHierarchy != actual.UseHierarchy {
|
||||
logrus.Printf("Expected memory use hierarchy %v, but found %v\n", expected.UseHierarchy, actual.UseHierarchy)
|
||||
t.Fail()
|
||||
t.Errorf("Expected memory use hierarchy %v, but found %v\n", expected.UseHierarchy, actual.UseHierarchy)
|
||||
}
|
||||
|
||||
for key, expValue := range expected.Stats {
|
||||
actValue, ok := actual.Stats[key]
|
||||
if !ok {
|
||||
logrus.Printf("Expected memory stat key %s not found\n", key)
|
||||
t.Fail()
|
||||
t.Errorf("Expected memory stat key %s not found\n", key)
|
||||
}
|
||||
if expValue != actValue {
|
||||
logrus.Printf("Expected memory stat value %d but found %d\n", expValue, actValue)
|
||||
t.Fail()
|
||||
t.Errorf("Expected memory stat value %d but found %d\n", expValue, actValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func expectMemoryDataEquals(t *testing.T, expected, actual cgroups.MemoryData) {
|
||||
if expected.Usage != actual.Usage {
|
||||
logrus.Printf("Expected memory usage %d but found %d\n", expected.Usage, actual.Usage)
|
||||
t.Fail()
|
||||
t.Errorf("Expected memory usage %d but found %d\n", expected.Usage, actual.Usage)
|
||||
}
|
||||
if expected.MaxUsage != actual.MaxUsage {
|
||||
logrus.Printf("Expected memory max usage %d but found %d\n", expected.MaxUsage, actual.MaxUsage)
|
||||
t.Fail()
|
||||
t.Errorf("Expected memory max usage %d but found %d\n", expected.MaxUsage, actual.MaxUsage)
|
||||
}
|
||||
if expected.Failcnt != actual.Failcnt {
|
||||
logrus.Printf("Expected memory failcnt %d but found %d\n", expected.Failcnt, actual.Failcnt)
|
||||
t.Fail()
|
||||
t.Errorf("Expected memory failcnt %d but found %d\n", expected.Failcnt, actual.Failcnt)
|
||||
}
|
||||
if expected.Limit != actual.Limit {
|
||||
logrus.Printf("Expected memory limit %d but found %d\n", expected.Limit, actual.Limit)
|
||||
t.Fail()
|
||||
t.Errorf("Expected memory limit %d but found %d\n", expected.Limit, actual.Limit)
|
||||
}
|
||||
}
|
||||
|
||||
func expectPageUsageByNUMAEquals(t *testing.T, expected, actual cgroups.PageUsageByNUMA) {
|
||||
if !reflect.DeepEqual(expected.Total, actual.Total) {
|
||||
t.Errorf("Expected total page usage by NUMA %#v but found %#v", expected.Total, actual.Total)
|
||||
}
|
||||
if !reflect.DeepEqual(expected.File, actual.File) {
|
||||
t.Errorf("Expected file page usage by NUMA %#v but found %#v", expected.File, actual.File)
|
||||
}
|
||||
if !reflect.DeepEqual(expected.Anon, actual.Anon) {
|
||||
t.Errorf("Expected anon page usage by NUMA %#v but found %#v", expected.Anon, actual.Anon)
|
||||
}
|
||||
if !reflect.DeepEqual(expected.Unevictable, actual.Unevictable) {
|
||||
t.Errorf("Expected unevictable page usage by NUMA %#v but found %#v", expected.Unevictable, actual.Unevictable)
|
||||
}
|
||||
if !reflect.DeepEqual(expected.Hierarchical.Total, actual.Hierarchical.Total) {
|
||||
t.Errorf("Expected hierarchical total page usage by NUMA %#v but found %#v", expected.Hierarchical.Total, actual.Hierarchical.Total)
|
||||
}
|
||||
if !reflect.DeepEqual(expected.Hierarchical.File, actual.Hierarchical.File) {
|
||||
t.Errorf("Expected hierarchical file page usage by NUMA %#v but found %#v", expected.Hierarchical.File, actual.Hierarchical.File)
|
||||
}
|
||||
if !reflect.DeepEqual(expected.Hierarchical.Anon, actual.Hierarchical.Anon) {
|
||||
t.Errorf("Expected hierarchical anon page usage by NUMA %#v but found %#v", expected.Hierarchical.Anon, actual.Hierarchical.Anon)
|
||||
}
|
||||
if !reflect.DeepEqual(expected.Hierarchical.Unevictable, actual.Hierarchical.Unevictable) {
|
||||
t.Errorf("Expected hierarchical total page usage by NUMA %#v but found %#v", expected.Hierarchical.Unevictable, actual.Hierarchical.Unevictable)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -51,12 +51,33 @@ type MemoryStats struct {
|
|||
KernelUsage MemoryData `json:"kernel_usage,omitempty"`
|
||||
// usage of kernel TCP memory
|
||||
KernelTCPUsage MemoryData `json:"kernel_tcp_usage,omitempty"`
|
||||
// usage of memory pages by NUMA node
|
||||
// see chapter 5.6 of memory controller documentation
|
||||
PageUsageByNUMA PageUsageByNUMA `json:"page_usage_by_numa,omitempty"`
|
||||
// if true, memory usage is accounted for throughout a hierarchy of cgroups.
|
||||
UseHierarchy bool `json:"use_hierarchy"`
|
||||
|
||||
Stats map[string]uint64 `json:"stats,omitempty"`
|
||||
}
|
||||
|
||||
type PageUsageByNUMA struct {
|
||||
// Embedding is used as types can't be recursive.
|
||||
PageUsageByNUMAInner
|
||||
Hierarchical PageUsageByNUMAInner `json:"hierarchical,omitempty"`
|
||||
}
|
||||
|
||||
type PageUsageByNUMAInner struct {
|
||||
Total PageStats `json:"total,omitempty"`
|
||||
File PageStats `json:"file,omitempty"`
|
||||
Anon PageStats `json:"anon,omitempty"`
|
||||
Unevictable PageStats `json:"unevictable,omitempty"`
|
||||
}
|
||||
|
||||
type PageStats struct {
|
||||
Total uint64 `json:"total,omitempty"`
|
||||
Nodes map[uint8]uint64 `json:"nodes,omitempty"`
|
||||
}
|
||||
|
||||
type PidsStats struct {
|
||||
// number of pids in the cgroup
|
||||
Current uint64 `json:"current,omitempty"`
|
||||
|
|
|
|||
Loading…
Reference in New Issue